diff --git a/.gitignore b/.gitignore index 7143eefa90d..12d7f55d162 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,6 @@ subprojects/*/ # clangd integration .cache/* + +# from gdb Python helper scripts +__pycache__ diff --git a/.gitmodules b/.gitmodules index 9a950a1ed72..e69de29bb2d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "subprojects/gst-integration-testsuites/medias"] - path = subprojects/gst-integration-testsuites/medias - url = https://gitlab.freedesktop.org/gstreamer/gst-integration-testsuites.git diff --git a/scripts/generate_init_static_plugins.py b/scripts/generate_init_static_plugins.py index f490aafc69d..377c9fd0ef2 100755 --- a/scripts/generate_init_static_plugins.py +++ b/scripts/generate_init_static_plugins.py @@ -2,6 +2,7 @@ import argparse import os +import platform from string import Template TEMPLATE = Template(''' @@ -93,7 +94,7 @@ def process_features(features_list, plugins, feature_prefix): giomodules_declaration = [] giomodules_registration = [] - if ',' in options.plugins or ':' in options.plugins: + if ',' in options.plugins or (platform.system() != 'Windows' and ':' in options.plugins): print("Only ';' is allowed in the list of plugins.") exit(1) diff --git a/subprojects/gst-integration-testsuites/medias b/subprojects/gst-integration-testsuites/medias deleted file mode 160000 index 121f8909493..00000000000 --- a/subprojects/gst-integration-testsuites/medias +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 121f8909493df214564c3db7fbba1dcb9217348e diff --git a/subprojects/gst-libav/ext/libav/gstavviddec.c b/subprojects/gst-libav/ext/libav/gstavviddec.c index 3763f56c6b5..d67cf4a4cf0 100644 --- a/subprojects/gst-libav/ext/libav/gstavviddec.c +++ b/subprojects/gst-libav/ext/libav/gstavviddec.c @@ -48,6 +48,7 @@ GST_DEBUG_CATEGORY_STATIC (GST_CAT_PERFORMANCE); #define DEFAULT_DIRECT_RENDERING TRUE #define DEFAULT_MAX_THREADS 0 #define DEFAULT_OUTPUT_CORRUPT TRUE +#define DEFAULT_REQUIRE_KEYFRAME FALSE #define REQUIRED_POOL_MAX_BUFFERS 32 #define DEFAULT_STRIDE_ALIGN 31 #define DEFAULT_ALLOC_PARAM { 0, DEFAULT_STRIDE_ALIGN, 0, 0, } @@ -65,6 +66,7 @@ enum PROP_OUTPUT_CORRUPT, PROP_THREAD_TYPE, PROP_STD_COMPLIANCE, + PROP_REQUIRE_KEYFRAME, PROP_LAST }; @@ -295,6 +297,11 @@ gst_ffmpegviddec_subclass_init (GstFFMpegVidDecClass * klass, g_param_spec_boolean ("output-corrupt", "Output corrupt buffers", "Whether libav should output frames even if corrupted", DEFAULT_OUTPUT_CORRUPT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_REQUIRE_KEYFRAME, + g_param_spec_boolean ("require-keyframe", "Require keyframe", + "Whether the first frame is required to be a keyframe", + DEFAULT_REQUIRE_KEYFRAME, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); caps = klass->in_plugin->capabilities; if (caps & (AV_CODEC_CAP_FRAME_THREADS | AV_CODEC_CAP_SLICE_THREADS)) { @@ -351,12 +358,14 @@ gst_ffmpegviddec_subinit (GstFFMpegVidDec * ffmpegdec) ffmpegdec->output_corrupt = DEFAULT_OUTPUT_CORRUPT; ffmpegdec->thread_type = DEFAULT_THREAD_TYPE; ffmpegdec->std_compliance = DEFAULT_STD_COMPLIANCE; + ffmpegdec->require_keyframe = DEFAULT_REQUIRE_KEYFRAME; GST_PAD_SET_ACCEPT_TEMPLATE (GST_VIDEO_DECODER_SINK_PAD (ffmpegdec)); gst_video_decoder_set_use_default_pad_acceptcaps (GST_VIDEO_DECODER_CAST (ffmpegdec), TRUE); gst_video_decoder_set_needs_format (GST_VIDEO_DECODER (ffmpegdec), TRUE); + gst_video_decoder_set_needs_sync_point (GST_VIDEO_DECODER (ffmpegdec), TRUE); } static void @@ -1792,6 +1801,8 @@ gst_ffmpegviddec_video_frame (GstFFMpegVidDec * ffmpegdec, } else if (res < 0) { GST_VIDEO_DECODER_ERROR (ffmpegdec, 1, STREAM, DECODE, (NULL), ("Video decoding error"), *ret); + gst_video_decoder_request_sync_point (GST_VIDEO_DECODER_CAST (ffmpegdec), + frame, GST_VIDEO_DECODER_REQUEST_SYNC_POINT_DISCARD_INPUT); goto beach; } @@ -1942,8 +1953,6 @@ gst_ffmpegviddec_video_frame (GstFFMpegVidDec * ffmpegdec, GST_TIME_FORMAT, tmp, tmp->system_frame_number, GST_TIME_ARGS (tmp->pts), GST_TIME_ARGS (tmp->dts)); /* drop extra ref and remove from frame list */ - GST_VIDEO_CODEC_FRAME_FLAG_UNSET (tmp, - GST_FFMPEG_VIDEO_CODEC_FRAME_FLAG_ALLOCATED); gst_video_decoder_release_frame (dec, tmp); } else { /* drop extra ref we got */ @@ -1954,15 +1963,31 @@ gst_ffmpegviddec_video_frame (GstFFMpegVidDec * ffmpegdec, g_list_free (ol); } - av_frame_unref (ffmpegdec->picture); + if (ffmpegdec->picture->key_frame || + ffmpegdec->picture->pict_type == AV_PICTURE_TYPE_I) + GST_VIDEO_CODEC_FRAME_SET_SYNC_POINT (out_frame); + else + GST_VIDEO_CODEC_FRAME_UNSET_SYNC_POINT (out_frame); if (frame) GST_VIDEO_CODEC_FRAME_FLAG_UNSET (frame, GST_FFMPEG_VIDEO_CODEC_FRAME_FLAG_ALLOCATED); + GST_LOG_OBJECT (ffmpegdec, "out_frame: pict_type %d, key_frame %d -> sync %d", + ffmpegdec->picture->pict_type, ffmpegdec->picture->key_frame, + GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (out_frame)); if (gst_video_decoder_get_subframe_mode (GST_VIDEO_DECODER (ffmpegdec))) gst_video_decoder_have_last_subframe (GST_VIDEO_DECODER (ffmpegdec), out_frame); + av_frame_unref (ffmpegdec->picture); + + if (ffmpegdec->requiring_keyframe) { + if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (out_frame)) + ffmpegdec->requiring_keyframe = FALSE; + else + gst_video_decoder_request_sync_point (GST_VIDEO_DECODER_CAST (ffmpegdec), + frame, GST_VIDEO_DECODER_REQUEST_SYNC_POINT_DISCARD_INPUT); + } /* FIXME: Ideally we would remap the buffer read-only now before pushing but * libav might still have a reference to it! @@ -2158,6 +2183,8 @@ gst_ffmpegviddec_handle_frame (GstVideoDecoder * decoder, */ GST_VIDEO_DECODER_STREAM_UNLOCK (ffmpegdec); if (avcodec_send_packet (ffmpegdec->context, &packet) < 0) { + gst_video_decoder_request_sync_point (GST_VIDEO_DECODER_CAST (ffmpegdec), + frame, GST_VIDEO_DECODER_REQUEST_SYNC_POINT_DISCARD_INPUT); GST_VIDEO_DECODER_STREAM_LOCK (ffmpegdec); av_packet_free_side_data (&packet); goto send_packet_failed; @@ -2209,6 +2236,8 @@ gst_ffmpegviddec_start (GstVideoDecoder * decoder) ffmpegdec->context->opaque = ffmpegdec; GST_OBJECT_UNLOCK (ffmpegdec); + ffmpegdec->requiring_keyframe = ffmpegdec->require_keyframe; + return TRUE; } @@ -2486,6 +2515,8 @@ gst_ffmpegviddec_set_property (GObject * object, break; case PROP_STD_COMPLIANCE: ffmpegdec->std_compliance = g_value_get_enum (value); + case PROP_REQUIRE_KEYFRAME: + ffmpegdec->require_keyframe = g_value_get_boolean (value); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -2526,6 +2557,9 @@ gst_ffmpegviddec_get_property (GObject * object, case PROP_STD_COMPLIANCE: g_value_set_enum (value, ffmpegdec->std_compliance); break; + case PROP_REQUIRE_KEYFRAME: + g_value_set_boolean (value, ffmpegdec->require_keyframe); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; diff --git a/subprojects/gst-libav/ext/libav/gstavviddec.h b/subprojects/gst-libav/ext/libav/gstavviddec.h index 0f713de5691..068aa203e42 100644 --- a/subprojects/gst-libav/ext/libav/gstavviddec.h +++ b/subprojects/gst-libav/ext/libav/gstavviddec.h @@ -94,6 +94,9 @@ struct _GstFFMpegVidDec gint pool_height; enum AVPixelFormat pool_format; GstVideoInfo pool_info; + + gboolean requiring_keyframe; + gboolean require_keyframe; }; typedef struct _GstFFMpegVidDecClass GstFFMpegVidDecClass; diff --git a/subprojects/gst-libav/ext/libav/gstavvidenc.c b/subprojects/gst-libav/ext/libav/gstavvidenc.c index fef5cae9041..6aaebdcca63 100644 --- a/subprojects/gst-libav/ext/libav/gstavvidenc.c +++ b/subprojects/gst-libav/ext/libav/gstavvidenc.c @@ -932,7 +932,7 @@ gst_ffmpegvidenc_start (GstVideoEncoder * encoder) return FALSE; } - gst_video_encoder_set_min_pts (encoder, GST_SECOND * 60 * 60 * 1000); + gst_video_encoder_set_min_pts (encoder, 0); return TRUE; } diff --git a/subprojects/gst-plugins-bad/ext/aes/meson.build b/subprojects/gst-plugins-bad/ext/aes/meson.build index 169fb83c49d..abdeeb33796 100644 --- a/subprojects/gst-plugins-bad/ext/aes/meson.build +++ b/subprojects/gst-plugins-bad/ext/aes/meson.build @@ -6,20 +6,22 @@ aes_sources = [ ] aes_cargs = [] -aes_dep = dependency('openssl', version : '>= 1.1.0', required : get_option('aes')) -if aes_dep.found() +openssl_dep = dependency('openssl', version : '>= 1.1.0', required : get_option('aes')) +if openssl_dep.found() aes_cargs += ['-DHAVE_OPENSSL'] else subdir_done() endif +winsock2_dep = cc.find_library('ws2_32', required : false) +gst_aes_deps = [gstpbutils_dep, gstvideo_dep, openssl_dep, winsock2_dep, gio_dep, libm] + gstaes = library('gstaes', aes_sources, c_args : gst_plugins_bad_args + aes_cargs, link_args : noseh_link_args, include_directories : [configinc], - dependencies : [gstpbutils_dep, gstvideo_dep, - aes_dep, gio_dep, libm], + dependencies : gst_aes_deps, install : true, install_dir : plugins_install_dir, ) diff --git a/subprojects/gst-plugins-bad/ext/opencv/gsthomography.cpp b/subprojects/gst-plugins-bad/ext/opencv/gsthomography.cpp new file mode 100644 index 00000000000..bdae2ad838c --- /dev/null +++ b/subprojects/gst-plugins-bad/ext/opencv/gsthomography.cpp @@ -0,0 +1,301 @@ +/* + * GStreamer + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Alternatively, the contents of this file may be used under the + * GNU Lesser General Public License Version 2.1 (the "LGPL"), in + * which case the following provisions apply instead of the ones + * mentioned above: + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include "gsthomography.h" +#include +#include + +GST_DEBUG_CATEGORY_STATIC (gst_homography_debug); +#define GST_CAT_DEFAULT gst_homography_debug + +/* Filter signals and args */ +enum +{ + /* FILL ME */ + LAST_SIGNAL +}; + +enum +{ + PROP_0, +}; + +/* the capabilities of the inputs and outputs. + * + * describe the real formats here. + */ +static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("RGB")) + ); + +static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("RGB")) + ); + +G_DEFINE_TYPE (GstHomography, gst_homography, GST_TYPE_OPENCV_VIDEO_FILTER); + +static void gst_homography_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_homography_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static GstFlowReturn gst_homography_transform (GstOpencvVideoFilter * filter, + GstBuffer * buf, IplImage * img, GstBuffer * outbuf, IplImage * outimg); + +static void +gst_homography_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstHomography *filter = GST_HOMOGRAPHY (object); + (void)filter; + (void)value; + + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_homography_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstHomography *filter = GST_HOMOGRAPHY (object); + (void)filter; + (void)value; + + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static GstCaps * +gst_homography_transform_caps (GstBaseTransform * trans, + GstPadDirection dir, GstCaps * caps, GstCaps * filter) +{ + GstCaps * ret = gst_caps_copy (caps); + guint i; + + GST_DEBUG_OBJECT (trans, + "transforming caps %"GST_PTR_FORMAT" filter %"GST_PTR_FORMAT" in direction %s", + caps, filter, (dir == GST_PAD_SINK) ? "sink" : "src"); + + for (i = 0; i < gst_caps_get_size (ret); ++i) { + GstStructure * s = gst_caps_get_structure (ret, i); + + gst_structure_set (s, + "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, + "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, NULL); + } + + if (!gst_caps_is_empty (ret) && filter) { + GstCaps *tmp = gst_caps_intersect_full (filter, ret, + GST_CAPS_INTERSECT_FIRST); + gst_caps_replace (&ret, tmp); + gst_caps_unref (tmp); + } + + GST_DEBUG_OBJECT (trans, "returning caps %" GST_PTR_FORMAT, ret); + + return ret; +} + + +static float +euclidean_distance (CvPoint2D32f * a, CvPoint2D32f * b) +{ + CvPoint2D32f d; + d.x = a->x - b->x; + d.y = a->y - b->y; + + return sqrtf (d.x * d.x + d.y * d.y); +} + +static GstCaps * +gst_homography_fixate_caps (GstBaseTransform * trans, + GstPadDirection dir, GstCaps * caps, GstCaps * othercaps) +{ + GstHomography *filter = GST_HOMOGRAPHY (trans); + GstCaps * ret; + GstStructure * out_s; + + g_return_val_if_fail (gst_caps_is_fixed (caps), NULL); + + GST_DEBUG_OBJECT (filter, + "fixating othercaps %" GST_PTR_FORMAT " in direction %s based on %"GST_PTR_FORMAT, + othercaps, (dir == GST_PAD_SINK) ? "sink" : "src", caps); + + ret = gst_caps_intersect (othercaps, caps); + if (gst_caps_is_empty (ret)) { + gst_caps_unref (ret); + ret = gst_caps_ref (othercaps); + } + + ret = gst_caps_make_writable (ret); + out_s = gst_caps_get_structure (ret, 0); + + float top = euclidean_distance (&filter->src_points[0], &filter->src_points[1]); + float right = euclidean_distance (&filter->src_points[1], &filter->src_points[2]); + float bottom = euclidean_distance (&filter->src_points[2], &filter->src_points[3]); + float left = euclidean_distance (&filter->src_points[3], &filter->src_points[0]); + + //float aspect_ratio = (top + bottom) / 2.0f + (right + left) / 2.0f; + + gint max_width = (gint)round (MAX (top, bottom)); + gint max_height = (gint)round (MAX (right, left)); + + gst_structure_set (out_s, + "width", G_TYPE_INT, max_width, + "height", G_TYPE_INT, max_height, + NULL); + + return ret; +} + +static GstFlowReturn +gst_homography_transform (GstOpencvVideoFilter * base, GstBuffer * buf, + IplImage * img, GstBuffer * outbuf, IplImage * outimg) +{ + GstHomography *filter = GST_HOMOGRAPHY (base); + + CvPoint2D32f dst_points[4]; + + dst_points[0].x = 0; + dst_points[0].y = 0; + dst_points[1].x = outimg->width - 1; + dst_points[1].y = 0; + dst_points[2].x = outimg->width - 1; + dst_points[2].y = outimg->height - 1; + dst_points[3].x = 0; + dst_points[3].y = outimg->height - 1; + + CvMat src_mat = cvMat (1, 4, CV_32FC2, &filter->src_points); + CvMat dst_mat = cvMat (1, 4, CV_32FC2, &dst_points); + + double H[9]; + CvMat res_mat = cvMat (3, 3, CV_64F, H); + + cvFindHomography(&src_mat, &dst_mat, &res_mat, CV_RANSAC); + + cvWarpPerspective (img, outimg, &res_mat); + + return GST_FLOW_OK; +} + + +/* initialize the homography's class */ +static void +gst_homography_class_init (GstHomographyClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstBaseTransformClass *trans_class = (GstBaseTransformClass *) klass; + GstOpencvVideoFilterClass *gstopencvbasefilter_class = (GstOpencvVideoFilterClass *) klass; + + gobject_class->set_property = gst_homography_set_property; + gobject_class->get_property = gst_homography_get_property; + + trans_class->transform_caps = GST_DEBUG_FUNCPTR (gst_homography_transform_caps); + trans_class->fixate_caps = GST_DEBUG_FUNCPTR (gst_homography_fixate_caps); + + gstopencvbasefilter_class->cv_trans_func = gst_homography_transform; + + gst_element_class_set_static_metadata (element_class, + "homography", + "Filter/Effect/Video", + "Performs homography.", + "Michael Sheldon "); + + gst_element_class_add_static_pad_template (element_class, &src_factory); + gst_element_class_add_static_pad_template (element_class, &sink_factory); +} + +/* initialize the new element + * instantiate pads and add them to element + * set pad calback functions + * initialize instance structure + */ +static void +gst_homography_init (GstHomography * filter) +{ + filter->src_points[0].x = 740; + filter->src_points[0].y = 187; + + filter->src_points[1].x = 1043; + filter->src_points[1].y = 165; + + filter->src_points[2].x = 1115; + filter->src_points[2].y = 604; + + filter->src_points[3].x = 771; + filter->src_points[3].y = 616; + + gst_opencv_video_filter_set_in_place (GST_OPENCV_VIDEO_FILTER_CAST (filter), + FALSE); +} + + +/* entry point to initialize the plug-in + * initialize the plug-in itself + * register the element factories and other features + */ +gboolean +gst_homography_plugin_init (GstPlugin * plugin) +{ + /* debug category for fltering log messages */ + GST_DEBUG_CATEGORY_INIT (gst_homography_debug, "homography", + 0, "Performs homography on videos and images"); + + return gst_element_register (plugin, "homography", GST_RANK_NONE, + GST_TYPE_HOMOGRAPHY); +} diff --git a/subprojects/gst-plugins-bad/ext/opencv/gsthomography.h b/subprojects/gst-plugins-bad/ext/opencv/gsthomography.h new file mode 100644 index 00000000000..44cd2430d54 --- /dev/null +++ b/subprojects/gst-plugins-bad/ext/opencv/gsthomography.h @@ -0,0 +1,81 @@ +/* + * GStreamer + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + * + * Alternatively, the contents of this file may be used under the + * GNU Lesser General Public License Version 2.1 (the "LGPL"), in + * which case the following provisions apply instead of the ones + * mentioned above: + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_HOMOGRAPHY_H__ +#define __GST_HOMOGRAPHY_H__ + +#include +#include + +G_BEGIN_DECLS +/* #defines don't like whitespacey bits */ +#define GST_TYPE_HOMOGRAPHY \ + (gst_homography_get_type()) +#define GST_HOMOGRAPHY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_HOMOGRAPHY,GstHomography)) +#define GST_HOMOGRAPHY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_HOMOGRAPHY,GstHomographyClass)) +#define GST_IS_HOMOGRAPHY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_HOMOGRAPHY)) +#define GST_IS_HOMOGRAPHY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_HOMOGRAPHY)) + +typedef struct _GstHomography GstHomography; +typedef struct _GstHomographyClass GstHomographyClass; + +struct _GstHomography +{ + GstOpencvVideoFilter element; + CvPoint2D32f src_points[4]; +}; + +struct _GstHomographyClass +{ + GstOpencvVideoFilterClass parent_class; +}; + +GType gst_homography_get_type (void); + +gboolean gst_homography_plugin_init (GstPlugin * plugin); + +G_END_DECLS +#endif /* __GST_HOMOGRAPHY_H__ */ diff --git a/subprojects/gst-plugins-bad/ext/opencv/gstopencv.cpp b/subprojects/gst-plugins-bad/ext/opencv/gstopencv.cpp index 5407a5a997c..70f85586bfc 100644 --- a/subprojects/gst-plugins-bad/ext/opencv/gstopencv.cpp +++ b/subprojects/gst-plugins-bad/ext/opencv/gstopencv.cpp @@ -32,6 +32,7 @@ #include "gstedgedetect.h" #include "gstfaceblur.h" #include "gstfacedetect.h" +#include "gsthomography.h" #include "gstmotioncells.h" #include "gsttemplatematch.h" #include "gsttextoverlay.h" @@ -61,6 +62,7 @@ plugin_init (GstPlugin * plugin) ret |= GST_ELEMENT_REGISTER (edgedetect, plugin); ret |= GST_ELEMENT_REGISTER (faceblur, plugin); ret |= GST_ELEMENT_REGISTER (facedetect, plugin); + ret |= GST_ELEMENT_REGISTER (homography, plugin); ret |= GST_ELEMENT_REGISTER (motioncells, plugin); ret |= GST_ELEMENT_REGISTER (templatematch, plugin); ret |= GST_ELEMENT_REGISTER (opencvtextoverlay, plugin); diff --git a/subprojects/gst-plugins-bad/ext/opencv/meson.build b/subprojects/gst-plugins-bad/ext/opencv/meson.build index 37e20156eb6..da88cff4a67 100644 --- a/subprojects/gst-plugins-bad/ext/opencv/meson.build +++ b/subprojects/gst-plugins-bad/ext/opencv/meson.build @@ -17,6 +17,7 @@ gstopencv_sources = [ 'gstfacedetect.cpp', 'gstgrabcut.cpp', 'gsthanddetect.cpp', + 'gsthomography.cpp', 'gstmotioncells.cpp', 'gstopencv.cpp', 'gstretinex.cpp', diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c index ec640ad0403..ff0bcaa305e 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.c @@ -1,6 +1,7 @@ /* * GStreamer * Copyright (C) 2010 Jan Schmidt + * Copyright (C) 2016 Pexip * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -49,16 +50,23 @@ #endif #include +#include +#include GST_DEBUG_CATEGORY_STATIC (gst_rtmp_sink_debug); #define GST_CAT_DEFAULT gst_rtmp_sink_debug +#define RTMP_LOCK(sink) g_mutex_lock (&(sink)->rtmp_lock) +#define RTMP_UNLOCK(sink) g_mutex_unlock (&(sink)->rtmp_lock) +#define RTMP_TRYLOCK(sink) g_mutex_trylock (&(sink)->rtmp_lock) + #define DEFAULT_LOCATION NULL enum { PROP_0, - PROP_LOCATION + PROP_LOCATION, + PROP_PACKETS_SENT }; static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", @@ -74,6 +82,7 @@ static void gst_rtmp_sink_set_property (GObject * object, guint prop_id, static void gst_rtmp_sink_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_rtmp_sink_finalize (GObject * object); +static gboolean gst_rtmp_sink_unlock (GstBaseSink * sink); static gboolean gst_rtmp_sink_stop (GstBaseSink * sink); static gboolean gst_rtmp_sink_start (GstBaseSink * sink); static gboolean gst_rtmp_sink_event (GstBaseSink * sink, GstEvent * event); @@ -107,6 +116,11 @@ gst_rtmp_sink_class_init (GstRTMPSinkClass * klass) g_param_spec_string ("location", "RTMP Location", "RTMP url", DEFAULT_LOCATION, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PACKETS_SENT, + g_param_spec_int ("packets-sent", "Packets sent", + "Number of packets sent (-1 = not available)", -1, G_MAXINT, -1, + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + gst_element_class_set_static_metadata (gstelement_class, "RTMP output sink", "Sink/Network", "Sends FLV content to a server via RTMP", @@ -116,6 +130,7 @@ gst_rtmp_sink_class_init (GstRTMPSinkClass * klass) gstbasesink_class->start = GST_DEBUG_FUNCPTR (gst_rtmp_sink_start); gstbasesink_class->stop = GST_DEBUG_FUNCPTR (gst_rtmp_sink_stop); + gstbasesink_class->unlock = GST_DEBUG_FUNCPTR (gst_rtmp_sink_unlock); gstbasesink_class->render = GST_DEBUG_FUNCPTR (gst_rtmp_sink_render); gstbasesink_class->set_caps = GST_DEBUG_FUNCPTR (gst_rtmp_sink_setcaps); gstbasesink_class->event = GST_DEBUG_FUNCPTR (gst_rtmp_sink_event); @@ -137,6 +152,8 @@ gst_rtmp_sink_init (GstRTMPSink * sink) GST_ERROR_OBJECT (sink, "WSAStartup failed: 0x%08x", WSAGetLastError ()); } #endif + + g_mutex_init (&sink->rtmp_lock); } static void @@ -148,6 +165,7 @@ gst_rtmp_sink_finalize (GObject * object) WSACleanup (); #endif g_free (sink->uri); + g_mutex_clear (&sink->rtmp_lock); G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -186,6 +204,7 @@ gst_rtmp_sink_start (GstBaseSink * basesink) sink->first = TRUE; sink->have_write_error = FALSE; + sink->connecting = FALSE; return TRUE; @@ -199,6 +218,43 @@ gst_rtmp_sink_start (GstBaseSink * basesink) return FALSE; } +static gboolean +gst_rtmp_sink_unlock (GstBaseSink * basesink) +{ + GstRTMPSink *sink = GST_RTMP_SINK (basesink); + + if (sink->rtmp == NULL) + return TRUE; + + /* Check to see if we currently are doing any activity towards librtmp */ + GST_DEBUG_OBJECT (sink, "Trying to lock"); + + if (!RTMP_TRYLOCK (sink)) { + GST_DEBUG_OBJECT (sink, "Lock NOT aquired..."); + /* if we are trying to connect, but the internal socket are not yet + initialized, we keep trying until either connection have failed or + the socket comes up */ + while (sink->connecting && sink->rtmp->m_sb.sb_socket == -1) { + g_thread_yield (); + } + + if (sink->rtmp->m_sb.sb_socket >= 0) { + GST_DEBUG_OBJECT (sink, "Shutting down internal librtmp socket"); + shutdown (sink->rtmp->m_sb.sb_socket, SHUT_RDWR); + } + + } else { + GST_DEBUG_OBJECT (sink, "Lock aquired..."); + RTMP_Close (sink->rtmp); + RTMP_Free (sink->rtmp); + sink->rtmp = NULL; + RTMP_UNLOCK (sink); + } + + return TRUE; +} + + static gboolean gst_rtmp_sink_stop (GstBaseSink * basesink) { @@ -228,30 +284,35 @@ gst_rtmp_sink_render (GstBaseSink * bsink, GstBuffer * buf) gboolean need_unref = FALSE; GstMapInfo map = GST_MAP_INFO_INIT; - if (sink->rtmp == NULL) { - /* Do not crash */ - GST_ELEMENT_ERROR (sink, RESOURCE, WRITE, (NULL), ("Failed to write data")); - return GST_FLOW_ERROR; - } - /* Ignore buffers that are in the stream headers (caps) */ if (GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_HEADER)) { return GST_FLOW_OK; } + if (sink->have_write_error) + goto write_failed; + if (sink->first) { /* open the connection */ - if (!RTMP_IsConnected (sink->rtmp)) { - if (!RTMP_Connect (sink->rtmp, NULL)) - goto connection_failed; + sink->first = FALSE; + sink->connecting = TRUE; + RTMP_LOCK (sink); - if (!RTMP_ConnectStream (sink->rtmp, 0)) { - RTMP_Close (sink->rtmp); + if (sink->rtmp == NULL) { + RTMP_UNLOCK (sink); + goto connection_failed; + } + if (!RTMP_IsConnected (sink->rtmp)) { + if (!RTMP_Connect (sink->rtmp, NULL) + || !RTMP_ConnectStream (sink->rtmp, 0)) { + RTMP_UNLOCK (sink); goto connection_failed; } GST_DEBUG_OBJECT (sink, "Opened connection to %s", sink->rtmp_uri); } + RTMP_UNLOCK (sink); + sink->connecting = FALSE; /* Prepend the header from the caps to the first non header buffer */ if (sink->header) { @@ -259,20 +320,19 @@ gst_rtmp_sink_render (GstBaseSink * bsink, GstBuffer * buf) gst_buffer_ref (buf)); need_unref = TRUE; } - - sink->first = FALSE; } - if (sink->have_write_error) - goto write_failed; - GST_LOG_OBJECT (sink, "Sending %" G_GSIZE_FORMAT " bytes to RTMP server", gst_buffer_get_size (buf)); gst_buffer_map (buf, &map, GST_MAP_READ); - if (RTMP_Write (sink->rtmp, (char *) map.data, map.size) <= 0) + RTMP_LOCK (sink); + if (sink->rtmp && RTMP_Write (sink->rtmp, (char *) map.data, map.size) <= 0) { + RTMP_UNLOCK (sink); goto write_failed; + } + RTMP_UNLOCK (sink); gst_buffer_unmap (buf, &map); if (need_unref) @@ -288,20 +348,16 @@ gst_rtmp_sink_render (GstBaseSink * bsink, GstBuffer * buf) if (need_unref) gst_buffer_unref (buf); sink->have_write_error = TRUE; - return GST_FLOW_ERROR; + return GST_FLOW_OK; } connection_failed: { GST_ELEMENT_ERROR (sink, RESOURCE, OPEN_WRITE, (NULL), ("Could not connect to RTMP stream \"%s\" for writing", sink->uri)); - RTMP_Free (sink->rtmp); - sink->rtmp = NULL; - g_free (sink->rtmp_uri); - sink->rtmp_uri = NULL; + sink->connecting = FALSE; sink->have_write_error = TRUE; - - return GST_FLOW_ERROR; + return GST_FLOW_OK; } } @@ -411,52 +467,43 @@ gst_rtmp_sink_setcaps (GstBaseSink * sink, GstCaps * caps) GstRTMPSink *rtmpsink = GST_RTMP_SINK (sink); GstStructure *s; const GValue *sh; + GArray *buffers; + gint i; GST_DEBUG_OBJECT (sink, "caps set to %" GST_PTR_FORMAT, caps); + s = gst_caps_get_structure (caps, 0); + sh = gst_structure_get_value (s, "streamheader"); + if (sh == NULL) { + GST_DEBUG_OBJECT (rtmpsink, "No streamheader in caps"); + return TRUE; + } + /* Clear our current header buffer */ if (rtmpsink->header) { gst_buffer_unref (rtmpsink->header); rtmpsink->header = NULL; } - s = gst_caps_get_structure (caps, 0); + rtmpsink->header = gst_buffer_new (); + buffers = g_value_peek_pointer (sh); - sh = gst_structure_get_value (s, "streamheader"); - if (sh == NULL) - goto out; + /* Concatenate all buffers in streamheader into one */ + for (i = 0; i < buffers->len; ++i) { + GValue *val; + GstBuffer *buf; - if (GST_VALUE_HOLDS_BUFFER (sh)) { - rtmpsink->header = gst_buffer_ref (gst_value_get_buffer (sh)); - } else if (GST_VALUE_HOLDS_ARRAY (sh)) { - GArray *buffers; - gint i; + val = &g_array_index (buffers, GValue, i); + buf = g_value_peek_pointer (val); - buffers = g_value_peek_pointer (sh); + gst_buffer_ref (buf); - /* Concatenate all buffers in streamheader into one */ - rtmpsink->header = gst_buffer_new (); - for (i = 0; i < buffers->len; ++i) { - GValue *val; - GstBuffer *buf; - - val = &g_array_index (buffers, GValue, i); - buf = g_value_peek_pointer (val); - - gst_buffer_ref (buf); - - rtmpsink->header = gst_buffer_append (rtmpsink->header, buf); - } - } else { - GST_ERROR_OBJECT (rtmpsink, "streamheader field has unexpected type %s", - G_VALUE_TYPE_NAME (sh)); + rtmpsink->header = gst_buffer_append (rtmpsink->header, buf); } GST_DEBUG_OBJECT (rtmpsink, "have %" G_GSIZE_FORMAT " bytes of header data", gst_buffer_get_size (rtmpsink->header)); -out: - return TRUE; } @@ -486,6 +533,12 @@ gst_rtmp_sink_get_property (GObject * object, guint prop_id, case PROP_LOCATION: g_value_set_string (value, sink->uri); break; + case PROP_PACKETS_SENT: + if (sink->rtmp) + g_value_set_int (value, sink->rtmp->m_nPacketsSent); + else + g_value_set_int (value, -1); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.h b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.h index daaa3bba774..c99c6acedf6 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.h +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsink.h @@ -52,6 +52,8 @@ struct _GstRTMPSink { RTMP *rtmp; gchar *rtmp_uri; /* copy of url for librtmp */ + GMutex rtmp_lock; + gboolean connecting; GstBuffer *header; gboolean first; diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c index f69b641b2d9..fc65ba47a7b 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.c @@ -5,6 +5,7 @@ * 2002,2003 Colin Walters * 2001,2010 Bastien Nocera * 2010 Sebastian Dröge + * 2016 Pexip * * rtmpsrc.c: * @@ -61,6 +62,8 @@ #include #include #include +#include +#include #include @@ -80,13 +83,20 @@ enum { PROP_0, PROP_LOCATION, - PROP_TIMEOUT + PROP_TIMEOUT, + PROP_BYTES_IN, + PROP_CLIENT_BW, + PROP_SERVER_BW, #if 0 - PROP_SWF_URL, + PROP_SWF_URL, PROP_PAGE_URL #endif }; +#define RTMP_LOCK(src) g_mutex_lock (&(src)->rtmp_lock) +#define RTMP_UNLOCK(src) g_mutex_unlock (&(src)->rtmp_lock) +#define RTMP_TRYLOCK(src) g_mutex_trylock (&(src)->rtmp_lock) + #define DEFAULT_LOCATION NULL #define DEFAULT_TIMEOUT 120 @@ -99,7 +109,6 @@ static void gst_rtmp_src_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_rtmp_src_finalize (GObject * object); -static gboolean gst_rtmp_src_connect (GstRTMPSrc * src); static gboolean gst_rtmp_src_unlock (GstBaseSrc * src); static gboolean gst_rtmp_src_stop (GstBaseSrc * src); static gboolean gst_rtmp_src_start (GstBaseSrc * src); @@ -147,6 +156,21 @@ gst_rtmp_src_class_init (GstRTMPSrcClass * klass) 0, G_MAXINT, DEFAULT_TIMEOUT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_BYTES_IN, + g_param_spec_int ("bytes-in", "Bytes in", + "Number of bytes received (-1 = not available)", -1, G_MAXINT, -1, + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_CLIENT_BW, + g_param_spec_int ("client-bandwidth", "Client bandwidth", + "Client bandwidth (-1 = not available)", -1, G_MAXINT, -1, + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_SERVER_BW, + g_param_spec_int ("server-bandwidth", "Server bandwidth", + "Server bandwidth (-1 = not available)", -1, G_MAXINT, -1, + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + gst_element_class_add_static_pad_template (gstelement_class, &srctemplate); gst_element_class_set_static_metadata (gstelement_class, @@ -184,6 +208,7 @@ gst_rtmp_src_init (GstRTMPSrc * rtmpsrc) rtmpsrc->last_timestamp = 0; rtmpsrc->timeout = DEFAULT_TIMEOUT; + g_mutex_init (&rtmpsrc->rtmp_lock); gst_base_src_set_format (GST_BASE_SRC (rtmpsrc), GST_FORMAT_TIME); } @@ -193,6 +218,7 @@ gst_rtmp_src_finalize (GObject * object) GstRTMPSrc *rtmpsrc = GST_RTMP_SRC (object); g_free (rtmpsrc->uri); + g_mutex_clear (&rtmpsrc->rtmp_lock); rtmpsrc->uri = NULL; #ifdef G_OS_WIN32 @@ -319,6 +345,24 @@ gst_rtmp_src_get_property (GObject * object, guint prop_id, GValue * value, case PROP_TIMEOUT: g_value_set_int (value, src->timeout); break; + case PROP_BYTES_IN: + if (src->rtmp) + g_value_set_int (value, src->rtmp->m_nBytesIn); + else + g_value_set_int (value, -1); + break; + case PROP_CLIENT_BW: + if (src->rtmp) + g_value_set_int (value, src->rtmp->m_nClientBW); + else + g_value_set_int (value, -1); + break; + case PROP_SERVER_BW: + if (src->rtmp) + g_value_set_int (value, src->rtmp->m_nServerBW); + else + g_value_set_int (value, -1); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -342,12 +386,25 @@ gst_rtmp_src_create (GstPushSrc * pushsrc, GstBuffer ** buffer) src = GST_RTMP_SRC (pushsrc); - g_return_val_if_fail (src->rtmp != NULL, GST_FLOW_ERROR); + if (src->first) { + /* open the connection */ + src->first = FALSE; + src->connecting = TRUE; + RTMP_LOCK (src); - if (!RTMP_IsConnected (src->rtmp)) { - GST_DEBUG_OBJECT (src, "reconnecting"); - if (!gst_rtmp_src_connect (src)) - return GST_FLOW_ERROR; + if (src->rtmp == NULL) { + RTMP_UNLOCK (src); + goto connect_error; + } + + if (!RTMP_IsConnected (src->rtmp)) { + if (!RTMP_Connect (src->rtmp, NULL)) { + RTMP_UNLOCK (src); + goto connect_error; + } + } + RTMP_UNLOCK (src); + src->connecting = FALSE; } size = GST_BASE_SRC_CAST (pushsrc)->blocksize; @@ -367,7 +424,11 @@ gst_rtmp_src_create (GstPushSrc * pushsrc, GstBuffer ** buffer) bsize = 0; while (todo > 0) { - int read = RTMP_Read (src->rtmp, (char *) data, todo); + gint read = 0; + RTMP_LOCK (src); + if (src->rtmp) + read = RTMP_Read (src->rtmp, (char *) data, todo); + RTMP_UNLOCK (src); if (G_UNLIKELY (read == 0 && todo == size)) goto eos; @@ -400,11 +461,15 @@ gst_rtmp_src_create (GstPushSrc * pushsrc, GstBuffer ** buffer) GST_BUFFER_OFFSET (buf) = src->cur_offset; src->cur_offset += size; - if (src->last_timestamp == GST_CLOCK_TIME_NONE) - src->last_timestamp = src->rtmp->m_mediaStamp * GST_MSECOND; - else - src->last_timestamp = - MAX (src->last_timestamp, src->rtmp->m_mediaStamp * GST_MSECOND); + RTMP_LOCK (src); + if (src->rtmp) { + if (src->last_timestamp == GST_CLOCK_TIME_NONE) + src->last_timestamp = src->rtmp->m_mediaStamp * GST_MSECOND; + else + src->last_timestamp = + MAX (src->last_timestamp, src->rtmp->m_mediaStamp * GST_MSECOND); + } + RTMP_UNLOCK (src); GST_LOG_OBJECT (src, "Created buffer of size %u at %" G_GINT64_FORMAT " with timestamp %" GST_TIME_FORMAT, size, GST_BUFFER_OFFSET (buf), @@ -416,6 +481,14 @@ gst_rtmp_src_create (GstPushSrc * pushsrc, GstBuffer ** buffer) return GST_FLOW_OK; + /* ERRORS */ +connect_error: + { + GST_ERROR_OBJECT (src, "Could not connect to RTMP stream \"%s\" for reading", + src->uri); + src->connecting = FALSE; + return GST_FLOW_EOS; + } read_failed: { gst_buffer_unmap (buf, &map); @@ -430,7 +503,7 @@ gst_rtmp_src_create (GstPushSrc * pushsrc, GstBuffer ** buffer) if (src->cur_offset == 0) { GST_ELEMENT_ERROR (src, RESOURCE, READ, (NULL), ("Failed to read any data from stream, check your URL")); - return GST_FLOW_ERROR; + return GST_FLOW_EOS; } else { GST_DEBUG_OBJECT (src, "Reading data gave EOS"); return GST_FLOW_EOS; @@ -563,24 +636,17 @@ gst_rtmp_src_do_seek (GstBaseSrc * basesrc, GstSegment * segment) return FALSE; } + src->discont = TRUE; + /* Initial seek */ if (src->cur_offset == 0 && segment->start == 0) - goto success; + return TRUE; if (!src->seekable) { GST_LOG_OBJECT (src, "Not a seekable stream"); return FALSE; } - /* If we have just disconnected in unlock(), we need to re-connect - * and also let librtmp read some data before sending a seek, - * otherwise it will stall. Calling create() does both. */ - if (!RTMP_IsConnected (src->rtmp)) { - GstBuffer *buffer = NULL; - gst_rtmp_src_create (GST_PUSH_SRC (basesrc), &buffer); - gst_buffer_replace (&buffer, NULL); - } - src->last_timestamp = GST_CLOCK_TIME_NONE; if (!RTMP_SendSeek (src->rtmp, segment->start / GST_MSECOND)) { GST_ERROR_OBJECT (src, "Seeking failed"); @@ -588,40 +654,16 @@ gst_rtmp_src_do_seek (GstBaseSrc * basesrc, GstSegment * segment) return FALSE; } -success: - /* This is set here so that the call to create() above doesn't clear it */ - src->discont = TRUE; - GST_DEBUG_OBJECT (src, "Seek to %" GST_TIME_FORMAT " successful", GST_TIME_ARGS (segment->start)); return TRUE; } -static gboolean -gst_rtmp_src_connect (GstRTMPSrc * src) -{ - RTMP_Init (src->rtmp); - src->rtmp->Link.timeout = src->timeout; - if (!RTMP_SetupURL (src->rtmp, src->uri)) { - GST_ELEMENT_ERROR (src, RESOURCE, OPEN_READ, (NULL), - ("Failed to setup URL '%s'", src->uri)); - return FALSE; - } - src->seekable = !(src->rtmp->Link.lFlags & RTMP_LF_LIVE); - GST_INFO_OBJECT (src, "seekable %d", src->seekable); - - /* open if required */ - if (!RTMP_IsConnected (src->rtmp)) { - if (!RTMP_Connect (src->rtmp, NULL)) { - GST_ELEMENT_ERROR (src, RESOURCE, OPEN_READ, (NULL), - ("Could not connect to RTMP stream \"%s\" for reading", src->uri)); - return FALSE; - } - } - - return TRUE; -} +#define STR2AVAL(av,str) G_STMT_START { \ + av.av_val = str; \ + av.av_len = strlen(av.av_val); \ +} G_STMT_END; /* open the file, do stuff necessary to go to PAUSED state */ static gboolean @@ -640,14 +682,24 @@ gst_rtmp_src_start (GstBaseSrc * basesrc) src->last_timestamp = 0; src->discont = TRUE; + src->first = TRUE; + src->connecting = FALSE; + src->rtmp = RTMP_Alloc (); if (!src->rtmp) { GST_ERROR_OBJECT (src, "Could not allocate librtmp's RTMP context"); goto error; } - if (!gst_rtmp_src_connect (src)) + RTMP_Init (src->rtmp); + src->rtmp->Link.timeout = src->timeout; + if (!RTMP_SetupURL (src->rtmp, src->uri)) { + GST_ELEMENT_ERROR (src, RESOURCE, OPEN_READ, (NULL), + ("Failed to setup URL '%s'", src->uri)); goto error; + } + src->seekable = !(src->rtmp->Link.lFlags & RTMP_LF_LIVE); + GST_INFO_OBJECT (src, "seekable %d", src->seekable); return TRUE; @@ -659,17 +711,39 @@ gst_rtmp_src_start (GstBaseSrc * basesrc) return FALSE; } +#undef STR2AVAL + static gboolean gst_rtmp_src_unlock (GstBaseSrc * basesrc) { - GstRTMPSrc *rtmpsrc = GST_RTMP_SRC (basesrc); + GstRTMPSrc *src = GST_RTMP_SRC (basesrc); - GST_DEBUG_OBJECT (rtmpsrc, "unlock"); + if (src->rtmp == NULL) + return TRUE; - /* This closes the socket, which means that any pending socket calls - * error out. */ - if (rtmpsrc->rtmp) { - RTMP_Close (rtmpsrc->rtmp); + /* Check to see if we currently are doing any activity towards librtmp */ + GST_DEBUG_OBJECT (src, "Trying to lock"); + + if (!RTMP_TRYLOCK (src)) { + GST_DEBUG_OBJECT (src, "Lock NOT aquired..."); + /* if we are trying to connect, but the internal socket are not yet + initialized, we keep trying until either connection have failed or + the socket comes up */ + while (src->connecting && src->rtmp->m_sb.sb_socket == -1) { + g_thread_yield (); + } + + if (src->rtmp->m_sb.sb_socket >= 0) { + GST_DEBUG_OBJECT (src, "Shutting down internal librtmp socket"); + shutdown (src->rtmp->m_sb.sb_socket, SHUT_RDWR); + } + + } else { + GST_DEBUG_OBJECT (src, "Lock aquired..."); + RTMP_Close (src->rtmp); + RTMP_Free (src->rtmp); + src->rtmp = NULL; + RTMP_UNLOCK (src); } return TRUE; @@ -684,6 +758,7 @@ gst_rtmp_src_stop (GstBaseSrc * basesrc) src = GST_RTMP_SRC (basesrc); if (src->rtmp) { + RTMP_Close (src->rtmp); RTMP_Free (src->rtmp); src->rtmp = NULL; } diff --git a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.h b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.h index 60ebcb226a3..b4f7dae2643 100644 --- a/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.h +++ b/subprojects/gst-plugins-bad/ext/rtmp/gstrtmpsrc.h @@ -64,6 +64,10 @@ struct _GstRTMPSrc RTMP *rtmp; int timeout; + GMutex rtmp_lock; + gboolean connecting; + gboolean first; + gint64 cur_offset; GstClockTime last_timestamp; gboolean seekable; diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c index a90f89428f2..5699e47b7c3 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.c @@ -53,6 +53,7 @@ GST_STATIC_PAD_TEMPLATE ("src_%u", GST_PAD_SRC, enum { SIGNAL_RESET_STREAM, + SIGNAL_ASSOC_RESTART, NUM_SIGNALS }; @@ -76,10 +77,14 @@ static GParamSpec *properties[NUM_PROPERTIES]; #define MAX_GST_SCTP_ASSOCIATION_ID 65535 #define MAX_STREAM_ID 65535 +#define GST_SCTP_DEC_GET_ASSOC_MUTEX(self) (&self->association_mutex) +#define GST_SCTP_DEC_ASSOC_MUTEX_LOCK(self) (g_mutex_lock (GST_SCTP_DEC_GET_ASSOC_MUTEX (self))) +#define GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK(self) (g_mutex_unlock (GST_SCTP_DEC_GET_ASSOC_MUTEX (self))) + GType gst_sctp_dec_pad_get_type (void); #define GST_TYPE_SCTP_DEC_PAD (gst_sctp_dec_pad_get_type()) -#define GST_SCTP_DEC_PAD(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_SCTP_DEC_PAD, GstSctpDecPad)) +#define GST_SCTP_DEC_PAD_CAST(obj) (GstSctpDecPad*)(obj) #define GST_SCTP_DEC_PAD_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_SCTP_DEC_PAD, GstSctpDecPadClass)) #define GST_IS_SCTP_DEC_PAD(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_SCTP_DEC_PAD)) #define GST_IS_SCTP_DEC_PAD_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_SCTP_DEC_PAD)) @@ -99,7 +104,7 @@ G_DEFINE_TYPE (GstSctpDecPad, gst_sctp_dec_pad, GST_TYPE_PAD); static void gst_sctp_dec_pad_finalize (GObject * object) { - GstSctpDecPad *self = GST_SCTP_DEC_PAD (object); + GstSctpDecPad *self = GST_SCTP_DEC_PAD_CAST (object); gst_object_unref (self->packet_queue); @@ -141,6 +146,7 @@ gst_sctp_dec_pad_init (GstSctpDecPad * self) data_queue_full_cb, data_queue_empty_cb, NULL); } +static void gst_sctp_dec_dispose (GObject * object); static void gst_sctp_dec_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_sctp_dec_get_property (GObject * object, guint prop_id, @@ -155,14 +161,14 @@ static gboolean gst_sctp_dec_packet_event (GstPad * pad, GstSctpDec * self, static void gst_sctp_data_srcpad_loop (GstPad * pad); static gboolean configure_association (GstSctpDec * self); +static void cleanup_association (GstSctpDec * self); static void on_gst_sctp_association_stream_reset (GstSctpAssociation * - gst_sctp_association, guint16 stream_id, GstSctpDec * self); + gst_sctp_association, guint16 stream_id, gpointer user_data); +static void on_gst_sctp_association_restart (GstSctpAssociation * + gst_sctp_association, gpointer user_data); static void on_receive (GstSctpAssociation * gst_sctp_association, guint8 * buf, gsize length, guint16 stream_id, guint ppid, gpointer user_data); -static void stop_srcpad_task (GstPad * pad); -static void stop_all_srcpad_tasks (GstSctpDec * self); -static void sctpdec_cleanup (GstSctpDec * self); static GstPad *get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id); static void remove_pad (GstSctpDec * self, GstPad * pad); static void on_reset_stream (GstSctpDec * self, guint stream_id); @@ -186,6 +192,7 @@ gst_sctp_dec_class_init (GstSctpDecClass * klass) gobject_class->set_property = gst_sctp_dec_set_property; gobject_class->get_property = gst_sctp_dec_get_property; + gobject_class->dispose = gst_sctp_dec_dispose; gobject_class->finalize = gst_sctp_dec_finalize; element_class->change_state = GST_DEBUG_FUNCPTR (gst_sctp_dec_change_state); @@ -215,6 +222,11 @@ gst_sctp_dec_class_init (GstSctpDecClass * klass) G_STRUCT_OFFSET (GstSctpDecClass, on_reset_stream), NULL, NULL, NULL, G_TYPE_NONE, 1, G_TYPE_UINT); + signals[SIGNAL_ASSOC_RESTART] = g_signal_new ("sctp-association-restarted", + G_TYPE_FROM_CLASS (gobject_class), G_SIGNAL_RUN_LAST, + G_STRUCT_OFFSET (GstSctpDecClass, on_association_restart), NULL, NULL, + g_cclosure_marshal_generic, G_TYPE_NONE, 0); + gst_element_class_set_static_metadata (element_class, "SCTP Decoder", "Decoder/Network/SCTP", @@ -225,9 +237,11 @@ gst_sctp_dec_class_init (GstSctpDecClass * klass) static void gst_sctp_dec_init (GstSctpDec * self) { + g_mutex_init (GST_SCTP_DEC_GET_ASSOC_MUTEX (self)); + + self->sctp_association = NULL; self->sctp_association_id = DEFAULT_GST_SCTP_ASSOCIATION_ID; self->local_sctp_port = DEFAULT_LOCAL_SCTP_PORT; - self->flow_combiner = gst_flow_combiner_new (); self->sink_pad = gst_pad_new_from_static_template (&sink_template, "sink"); @@ -239,11 +253,63 @@ gst_sctp_dec_init (GstSctpDec * self) gst_element_add_pad (GST_ELEMENT (self), self->sink_pad); } +static void +remove_pad (GstSctpDec * self, GstPad * pad) +{ + GST_DEBUG_OBJECT (pad, "Removing pad"); + + gst_pad_set_active (pad, FALSE); + if (gst_object_has_as_parent (GST_OBJECT (pad), GST_OBJECT (self))) + gst_element_remove_pad (GST_ELEMENT (self), pad); + + GST_OBJECT_LOCK (self); + gst_flow_combiner_remove_pad (self->flow_combiner, pad); + GST_OBJECT_UNLOCK (self); +} + +static void +remove_pad_it (const GValue * item, gpointer user_data) +{ + GstPad *pad = g_value_get_object (item); + GstSctpDec *self = user_data; + + remove_pad (self, pad); +} + +static void +gst_sctp_dec_dispose (GObject * object) +{ + GstSctpDec *self = GST_SCTP_DEC_CAST (object); + GstIterator *it; + + /* remove all srcpads */ + it = gst_element_iterate_src_pads (GST_ELEMENT (self)); + while (gst_iterator_foreach (it, remove_pad_it, self) == GST_ITERATOR_RESYNC) + gst_iterator_resync (it); + gst_iterator_free (it); + + + G_OBJECT_CLASS (gst_sctp_dec_parent_class)->dispose (object); +} + +static void +gst_sctp_dec_finalize (GObject * object) +{ + GstSctpDec *self = GST_SCTP_DEC_CAST (object); + + gst_flow_combiner_free (self->flow_combiner); + self->flow_combiner = NULL; + + g_mutex_clear (GST_SCTP_DEC_GET_ASSOC_MUTEX (self)); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + static void gst_sctp_dec_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { - GstSctpDec *self = GST_SCTP_DEC (object); + GstSctpDec *self = GST_SCTP_DEC_CAST (object); switch (prop_id) { case PROP_GST_SCTP_ASSOCIATION_ID: @@ -251,6 +317,13 @@ gst_sctp_dec_set_property (GObject * object, guint prop_id, break; case PROP_LOCAL_SCTP_PORT: self->local_sctp_port = g_value_get_uint (value); + + GST_SCTP_DEC_ASSOC_MUTEX_LOCK (self); + if (self->sctp_association) { + g_object_set (self->sctp_association, "local-port", + self->local_sctp_port, NULL); + } + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (self, prop_id, pspec); @@ -262,7 +335,7 @@ static void gst_sctp_dec_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { - GstSctpDec *self = GST_SCTP_DEC (object); + GstSctpDec *self = GST_SCTP_DEC_CAST (object); switch (prop_id) { case PROP_GST_SCTP_ASSOCIATION_ID: @@ -277,42 +350,22 @@ gst_sctp_dec_get_property (GObject * object, guint prop_id, GValue * value, } } -static void -gst_sctp_dec_finalize (GObject * object) -{ - GstSctpDec *self = GST_SCTP_DEC (object); - - gst_flow_combiner_free (self->flow_combiner); - self->flow_combiner = NULL; - - G_OBJECT_CLASS (parent_class)->finalize (object); -} - static GstStateChangeReturn gst_sctp_dec_change_state (GstElement * element, GstStateChange transition) { - GstSctpDec *self = GST_SCTP_DEC (element); - GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; + GstSctpDec *self = GST_SCTP_DEC_CAST (element); + GstStateChangeReturn ret; + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); switch (transition) { - case GST_STATE_CHANGE_READY_TO_PAUSED: + case GST_STATE_CHANGE_NULL_TO_READY: gst_flow_combiner_reset (self->flow_combiner); if (!configure_association (self)) ret = GST_STATE_CHANGE_FAILURE; break; - case GST_STATE_CHANGE_PAUSED_TO_READY: - stop_all_srcpad_tasks (self); - break; - default: - break; - } - - if (ret != GST_STATE_CHANGE_FAILURE) - ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); - - switch (transition) { - case GST_STATE_CHANGE_PAUSED_TO_READY: - sctpdec_cleanup (self); + case GST_STATE_CHANGE_READY_TO_NULL: + cleanup_association (self); gst_flow_combiner_reset (self->flow_combiner); break; default: @@ -327,6 +380,7 @@ gst_sctp_dec_packet_chain (GstPad * pad, GstSctpDec * self, GstBuffer * buf) { GstFlowReturn flow_ret; GstMapInfo map; + GstSctpAssociation *sctp_association = NULL; GST_DEBUG_OBJECT (self, "Processing received buffer %" GST_PTR_FORMAT, buf); @@ -336,8 +390,16 @@ gst_sctp_dec_packet_chain (GstPad * pad, GstSctpDec * self, GstBuffer * buf) return GST_FLOW_ERROR; } - gst_sctp_association_incoming_packet (self->sctp_association, - (const guint8 *) map.data, (guint32) map.size); + GST_SCTP_DEC_ASSOC_MUTEX_LOCK (self); + sctp_association = gst_sctp_association_ref (self->sctp_association); + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); + + if (sctp_association) { + gst_sctp_association_incoming_packet (sctp_association, + (const guint8 *) map.data, (guint32) map.size); + gst_sctp_association_unref (sctp_association); + } + gst_buffer_unmap (buf, &map); gst_buffer_unref (buf); @@ -364,8 +426,6 @@ flush_srcpad (const GValue * item, gpointer user_data) gst_data_queue_flush (sctpdec_pad->packet_queue); } else { gst_data_queue_set_flushing (sctpdec_pad->packet_queue, FALSE); - gst_pad_start_task (GST_PAD (sctpdec_pad), - (GstTaskFunction) gst_sctp_data_srcpad_loop, sctpdec_pad, NULL); } } @@ -375,8 +435,9 @@ gst_sctp_dec_packet_event (GstPad * pad, GstSctpDec * self, GstEvent * event) switch (GST_EVENT_TYPE (event)) { case GST_EVENT_STREAM_START: case GST_EVENT_CAPS: - /* We create our own stream-start events and the caps event does not - * make sense */ + case GST_EVENT_SEGMENT: + /* We create our own stream-start and segment events and the + * caps event does not make sense */ gst_event_unref (event); return TRUE; case GST_EVENT_EOS: @@ -414,10 +475,10 @@ static void gst_sctp_data_srcpad_loop (GstPad * pad) { GstSctpDec *self; - GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD (pad); + GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD_CAST (pad); GstDataQueueItem *item; - self = GST_SCTP_DEC (gst_pad_get_parent (pad)); + self = GST_SCTP_DEC_CAST (gst_pad_get_parent (pad)); if (gst_data_queue_pop (sctpdec_pad->packet_queue, &item)) { GstBuffer *buffer; @@ -467,32 +528,33 @@ static gboolean configure_association (GstSctpDec * self) { gint state; + GstSctpAssociationDecoderCtx ctx; + GST_SCTP_DEC_ASSOC_MUTEX_LOCK (self); self->sctp_association = gst_sctp_association_get (self->sctp_association_id); - g_object_get (self->sctp_association, "state", &state, NULL); if (state != GST_SCTP_ASSOCIATION_STATE_NEW) { GST_WARNING_OBJECT (self, "Could not configure SCTP association. Association already in use!"); - g_object_unref (self->sctp_association); + gst_sctp_association_unref (self->sctp_association); self->sctp_association = NULL; - goto error; + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); + return FALSE; } - self->signal_handler_stream_reset = - g_signal_connect_object (self->sctp_association, "stream-reset", - G_CALLBACK (on_gst_sctp_association_stream_reset), self, 0); + g_object_set (self->sctp_association, "local-port", self->local_sctp_port, + NULL); - g_object_bind_property (self, "local-sctp-port", self->sctp_association, - "local-port", G_BINDING_SYNC_CREATE); + ctx.element = self; + ctx.stream_reset_cb = on_gst_sctp_association_stream_reset; + ctx.restart_cb = on_gst_sctp_association_restart; + ctx.packet_received_cb = on_receive; + gst_sctp_association_set_decoder_ctx (self->sctp_association, &ctx); - gst_sctp_association_set_on_packet_received (self->sctp_association, - on_receive, gst_object_ref (self), gst_object_unref); + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); return TRUE; -error: - return FALSE; } static gboolean @@ -501,17 +563,14 @@ gst_sctp_dec_src_event (GstPad * pad, GstSctpDec * self, GstEvent * event) switch (GST_EVENT_TYPE (event)) { case GST_EVENT_RECONFIGURE: case GST_EVENT_FLUSH_STOP:{ - GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD (pad); + GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD_CAST (pad); - /* Unflush and start task again */ gst_data_queue_set_flushing (sctpdec_pad->packet_queue, FALSE); - gst_pad_start_task (pad, (GstTaskFunction) gst_sctp_data_srcpad_loop, pad, - NULL); return gst_pad_event_default (pad, GST_OBJECT (self), event); } case GST_EVENT_FLUSH_START:{ - GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD (pad); + GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD_CAST (pad); gst_data_queue_set_flushing (sctpdec_pad->packet_queue, TRUE); gst_data_queue_flush (sctpdec_pad->packet_queue); @@ -524,15 +583,52 @@ gst_sctp_dec_src_event (GstPad * pad, GstSctpDec * self, GstEvent * event) } static gboolean -copy_sticky_events (GstPad * pad, GstEvent ** event, gpointer user_data) +gst_sctp_dec_src_activate_mode (GstPad * pad, GstObject * parent, + GstPadMode mode, gboolean active) { - GstPad *new_pad = user_data; + GstSctpDec *self = GST_SCTP_DEC_CAST (parent); + GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD_CAST (pad); + gboolean result; - if (GST_EVENT_TYPE (*event) != GST_EVENT_CAPS - && GST_EVENT_TYPE (*event) != GST_EVENT_STREAM_START) - gst_pad_store_sticky_event (new_pad, *event); + GST_DEBUG_OBJECT (self, "activate mode %d active %d", mode, active); - return TRUE; + if (active) { + gst_data_queue_set_flushing (sctpdec_pad->packet_queue, FALSE); + result = + gst_pad_start_task (pad, (GstTaskFunction) gst_sctp_data_srcpad_loop, + pad, NULL); + } else { + gst_data_queue_set_flushing (sctpdec_pad->packet_queue, TRUE); + gst_data_queue_flush (sctpdec_pad->packet_queue); + result = gst_pad_stop_task (pad); + } + + return result; +} + +static void +send_sticky_events (GstSctpDec * self, GstPad * pad, guint16 stream_id) +{ + GstSegment segment; + gchar *pad_stream_id; + gboolean ret; + + pad_stream_id = + gst_pad_create_stream_id_printf (pad, GST_ELEMENT (self), "%hu", + stream_id); + ret = gst_pad_push_event (pad, gst_event_new_stream_start (pad_stream_id)); + g_free (pad_stream_id); + if (ret == FALSE) { + GST_ERROR_OBJECT (self, + "Pushing stream-start event failed on pad %" GST_PTR_FORMAT, pad); + } + + gst_segment_init (&segment, GST_FORMAT_TIME); + ret = gst_pad_push_event (pad, gst_event_new_segment (&segment)); + if (ret == FALSE) { + GST_ERROR_OBJECT (self, + "Pushing segment event failed on pad %" GST_PTR_FORMAT, pad); + } } static GstPad * @@ -540,7 +636,7 @@ get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id) { GstPad *new_pad = NULL; gint state; - gchar *pad_name, *pad_stream_id; + gchar *pad_name; GstPadTemplate *template; pad_name = g_strdup_printf ("src_%hu", stream_id); @@ -550,11 +646,20 @@ get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id) return new_pad; } + GST_SCTP_DEC_ASSOC_MUTEX_LOCK (self); + if (!self->sctp_association) { + GST_ERROR_OBJECT (self, "Attempt to get pad without a GstSctpAssociation"); + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); + return NULL; + } + g_object_get (self->sctp_association, "state", &state, NULL); + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); if (state != GST_SCTP_ASSOCIATION_STATE_CONNECTED) { GST_ERROR_OBJECT (self, - "The SCTP association must be established before a new stream can be created"); + "The SCTP association must be established before a new stream can be created (state: %d)", + state); return NULL; } @@ -571,27 +676,22 @@ get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id) gst_pad_set_event_function (new_pad, GST_DEBUG_FUNCPTR ((GstPadEventFunction) gst_sctp_dec_src_event)); + gst_pad_set_activatemode_function (new_pad, + GST_DEBUG_FUNCPTR (gst_sctp_dec_src_activate_mode)); + + if (!gst_element_add_pad (GST_ELEMENT (self), new_pad)) + goto error_add; if (!gst_pad_set_active (new_pad, TRUE)) goto error_cleanup; - pad_stream_id = - gst_pad_create_stream_id_printf (new_pad, GST_ELEMENT (self), "%hu", - stream_id); - gst_pad_push_event (new_pad, gst_event_new_stream_start (pad_stream_id)); - g_free (pad_stream_id); - gst_pad_sticky_events_foreach (self->sink_pad, copy_sticky_events, new_pad); + send_sticky_events (self, new_pad, stream_id); - if (!gst_element_add_pad (GST_ELEMENT (self), new_pad)) - goto error_add; GST_OBJECT_LOCK (self); gst_flow_combiner_add_pad (self->flow_combiner, new_pad); GST_OBJECT_UNLOCK (self); - gst_pad_start_task (new_pad, (GstTaskFunction) gst_sctp_data_srcpad_loop, - new_pad, NULL); - gst_object_ref (new_pad); return new_pad; @@ -602,26 +702,13 @@ get_pad_for_stream_id (GstSctpDec * self, guint16 stream_id) return NULL; } -static void -remove_pad (GstSctpDec * self, GstPad * pad) -{ - stop_srcpad_task (pad); - GST_PAD_STREAM_LOCK (pad); - gst_pad_set_active (pad, FALSE); - if (gst_object_has_as_parent (GST_OBJECT (pad), GST_OBJECT (self))) - gst_element_remove_pad (GST_ELEMENT (self), pad); - GST_PAD_STREAM_UNLOCK (pad); - GST_OBJECT_LOCK (self); - gst_flow_combiner_remove_pad (self->flow_combiner, pad); - GST_OBJECT_UNLOCK (self); -} - static void on_gst_sctp_association_stream_reset (GstSctpAssociation * gst_sctp_association, - guint16 stream_id, GstSctpDec * self) + guint16 stream_id, gpointer user_data) { gchar *pad_name; GstPad *srcpad; + GstSctpDec *self = user_data; GST_DEBUG_OBJECT (self, "Stream %u reset", stream_id); @@ -642,6 +729,15 @@ on_gst_sctp_association_stream_reset (GstSctpAssociation * gst_sctp_association, gst_object_unref (srcpad); } +static void +on_gst_sctp_association_restart (GstSctpAssociation * gst_sctp_association, + gpointer user_data) +{ + GstSctpDec *self = user_data; + (void) gst_sctp_association; + g_signal_emit (self, signals[SIGNAL_ASSOC_RESTART], 0); +} + static void data_queue_item_free (GstDataQueueItem * item) { @@ -661,13 +757,15 @@ on_receive (GstSctpAssociation * sctp_association, guint8 * buf, GstBuffer *gstbuf; src_pad = get_pad_for_stream_id (self, stream_id); - g_assert (src_pad); + /* If we don't have a src_pad it could mean the association is disconnecting */ + if (!src_pad) + return; GST_DEBUG_OBJECT (src_pad, "Received incoming packet of size %" G_GSIZE_FORMAT " with stream id %u ppid %u", length, stream_id, ppid); - sctpdec_pad = GST_SCTP_DEC_PAD (src_pad); + sctpdec_pad = GST_SCTP_DEC_PAD_CAST (src_pad); gstbuf = gst_buffer_new_wrapped_full (0, buf, length, 0, length, buf, (GDestroyNotify) usrsctp_freedumpbuffer); @@ -687,55 +785,36 @@ on_receive (GstSctpAssociation * sctp_association, guint8 * buf, } static void -stop_srcpad_task (GstPad * pad) +cleanup_association (GstSctpDec * self) { - GstSctpDecPad *sctpdec_pad = GST_SCTP_DEC_PAD (pad); + GstSctpAssociationDecoderCtx ctx; - gst_data_queue_set_flushing (sctpdec_pad->packet_queue, TRUE); - gst_data_queue_flush (sctpdec_pad->packet_queue); - gst_pad_stop_task (pad); -} - -static void -remove_pad_it (const GValue * item, gpointer user_data) -{ - GstPad *pad = g_value_get_object (item); - GstSctpDec *self = user_data; - - remove_pad (self, pad); -} - -static void -stop_all_srcpad_tasks (GstSctpDec * self) -{ - GstIterator *it; - - it = gst_element_iterate_src_pads (GST_ELEMENT (self)); - while (gst_iterator_foreach (it, remove_pad_it, self) == GST_ITERATOR_RESYNC) - gst_iterator_resync (it); - gst_iterator_free (it); -} - -static void -sctpdec_cleanup (GstSctpDec * self) -{ - if (self->sctp_association) { - gst_sctp_association_set_on_packet_received (self->sctp_association, NULL, - NULL, NULL); - g_signal_handler_disconnect (self->sctp_association, - self->signal_handler_stream_reset); - gst_sctp_association_force_close (self->sctp_association); - g_object_unref (self->sctp_association); - self->sctp_association = NULL; + GST_SCTP_DEC_ASSOC_MUTEX_LOCK (self); + if (!self->sctp_association) { + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); + return; } + + memset (&ctx, 0, sizeof (GstSctpAssociationDecoderCtx)); + gst_sctp_association_set_decoder_ctx (self->sctp_association, &ctx); + gst_sctp_association_force_close (self->sctp_association); + gst_sctp_association_unref (self->sctp_association); + self->sctp_association = NULL; + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); } static void on_reset_stream (GstSctpDec * self, guint stream_id) { - if (self->sctp_association) { - gst_sctp_association_reset_stream (self->sctp_association, stream_id); - on_gst_sctp_association_stream_reset (self->sctp_association, stream_id, - self); + GstSctpAssociation *sctp_association = NULL; + + GST_SCTP_DEC_ASSOC_MUTEX_LOCK (self); + sctp_association = gst_sctp_association_ref (self->sctp_association); + GST_SCTP_DEC_ASSOC_MUTEX_UNLOCK (self); + + if (sctp_association) { + gst_sctp_association_reset_stream (sctp_association, stream_id); + on_gst_sctp_association_stream_reset (sctp_association, stream_id, self); + gst_sctp_association_unref (sctp_association); } } diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h index c6c89865718..c431e41b171 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpdec.h @@ -35,6 +35,7 @@ G_BEGIN_DECLS #define GST_TYPE_SCTP_DEC (gst_sctp_dec_get_type()) #define GST_SCTP_DEC(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_SCTP_DEC, GstSctpDec)) +#define GST_SCTP_DEC_CAST(obj) (GstSctpDec*)(obj) #define GST_SCTP_DEC_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_SCTP_DEC, GstSctpDecClass)) #define GST_IS_SCTP_DEC(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_SCTP_DEC)) #define GST_IS_SCTP_DEC_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_SCTP_DEC)) @@ -45,6 +46,8 @@ struct _GstSctpDec { GstElement element; + GMutex association_mutex; + GstFlowCombiner *flow_combiner; GstPad *sink_pad; @@ -52,7 +55,6 @@ struct _GstSctpDec guint local_sctp_port; GstSctpAssociation *sctp_association; - gulong signal_handler_stream_reset; }; struct _GstSctpDecClass @@ -60,6 +62,7 @@ struct _GstSctpDecClass GstElementClass parent_class; void (*on_reset_stream) (GstSctpDec * sctp_dec, guint stream_id); + void (*on_association_restart) (GstSctpDec * sctp_dec); }; GType gst_sctp_dec_get_type (void); diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c index 2fcbecebb9f..604b3c162cc 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.c @@ -51,6 +51,8 @@ enum { SIGNAL_SCTP_ASSOCIATION_ESTABLISHED, SIGNAL_GET_STREAM_BYTES_SENT, + SIGNAL_DISCONNECT, + SIGNAL_RECONNECT, NUM_SIGNALS }; @@ -63,6 +65,7 @@ enum PROP_GST_SCTP_ASSOCIATION_ID, PROP_REMOTE_SCTP_PORT, PROP_USE_SOCK_STREAM, + PROP_AGGRESSIVE_HEARTBEAT, NUM_PROPERTIES }; @@ -77,11 +80,15 @@ static GParamSpec *properties[NUM_PROPERTIES]; #define BUFFER_FULL_SLEEP_TIME 100000 +#define GST_SCTP_ENC_GET_ASSOC_MUTEX(self) (&self->association_mutex) +#define GST_SCTP_ENC_ASSOC_MUTEX_LOCK(self) (g_mutex_lock (GST_SCTP_ENC_GET_ASSOC_MUTEX (self))) +#define GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK(self) (g_mutex_unlock (GST_SCTP_ENC_GET_ASSOC_MUTEX (self))) + GType gst_sctp_enc_pad_get_type (void); #define GST_TYPE_SCTP_ENC_PAD (gst_sctp_enc_pad_get_type()) -#define GST_SCTP_ENC_PAD(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_SCTP_ENC_PAD, GstSctpEncPad)) -#define GST_SCTP_ENC_PAD_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_SCTP_ENC_PAD, GstSctpEncPadClass)) +#define GST_SCTP_ENC_PAD_CAST(obj) (GstSctpEncPad*)(obj) +#define GST_SCTP_ENC_PAD_CAST_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_SCTP_ENC_PAD, GstSctpEncPadClass)) #define GST_IS_SCTP_ENC_PAD(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_SCTP_ENC_PAD)) #define GST_IS_SCTP_ENC_PAD_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_SCTP_ENC_PAD)) @@ -111,7 +118,7 @@ G_DEFINE_TYPE (GstSctpEncPad, gst_sctp_enc_pad, GST_TYPE_PAD); static void gst_sctp_enc_pad_finalize (GObject * object) { - GstSctpEncPad *self = GST_SCTP_ENC_PAD (object); + GstSctpEncPad *self = GST_SCTP_ENC_PAD_CAST (object); g_cond_clear (&self->cond); g_mutex_clear (&self->lock); @@ -136,6 +143,7 @@ gst_sctp_enc_pad_init (GstSctpEncPad * self) self->clear_to_send = FALSE; } +static void gst_sctp_enc_dispose (GObject * object); static void gst_sctp_enc_finalize (GObject * object); static void gst_sctp_enc_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); @@ -153,18 +161,23 @@ static gboolean gst_sctp_enc_sink_event (GstPad * pad, GstObject * parent, GstEvent * event); static gboolean gst_sctp_enc_src_event (GstPad * pad, GstObject * parent, GstEvent * event); +static gboolean +gst_sctp_enc_src_activate_mode (GstPad * pad, GstObject * parent, + GstPadMode mode, gboolean active); static void on_sctp_association_state_changed (GstSctpAssociation * - sctp_association, GParamSpec * pspec, GstSctpEnc * self); + sctp_association, GstSctpAssociationState state, gpointer user_data); static gboolean configure_association (GstSctpEnc * self); +static void cleanup_association (GstSctpEnc * self); + static void on_sctp_packet_out (GstSctpAssociation * sctp_association, const guint8 * buf, gsize length, gpointer user_data); -static void stop_srcpad_task (GstPad * pad, GstSctpEnc * self); -static void sctpenc_cleanup (GstSctpEnc * self); static void get_config_from_caps (const GstCaps * caps, gboolean * ordered, GstSctpAssociationPartialReliability * reliability, guint32 * reliability_param, guint32 * ppid, gboolean * ppid_available); static guint64 on_get_stream_bytes_sent (GstSctpEnc * self, guint stream_id); +static gboolean disconnect (GstSctpEnc * self); +static void reconnect (GstSctpEnc * self); static void gst_sctp_enc_class_init (GstSctpEncClass * klass) @@ -183,6 +196,7 @@ gst_sctp_enc_class_init (GstSctpEncClass * klass) gst_element_class_add_pad_template (GST_ELEMENT_CLASS (klass), gst_static_pad_template_get (&sink_template)); + gobject_class->dispose = GST_DEBUG_FUNCPTR (gst_sctp_enc_dispose); gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_sctp_enc_finalize); gobject_class->set_property = GST_DEBUG_FUNCPTR (gst_sctp_enc_set_property); gobject_class->get_property = GST_DEBUG_FUNCPTR (gst_sctp_enc_get_property); @@ -215,6 +229,11 @@ gst_sctp_enc_class_init (GstSctpEncClass * klass) "When TRUE the partial reliability parameters of the channel are ignored.", DEFAULT_USE_SOCK_STREAM, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + properties[PROP_AGGRESSIVE_HEARTBEAT] = + g_param_spec_boolean ("aggressive-heartbeat", "Aggressive heartbeat", + "When set to TRUE, set the heartbeat interval to 1000ms and the assoc " + "rtx max to 2.", FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + g_object_class_install_properties (gobject_class, NUM_PROPERTIES, properties); signals[SIGNAL_SCTP_ASSOCIATION_ESTABLISHED] = @@ -228,8 +247,20 @@ gst_sctp_enc_class_init (GstSctpEncClass * klass) G_STRUCT_OFFSET (GstSctpEncClass, on_get_stream_bytes_sent), NULL, NULL, NULL, G_TYPE_UINT64, 1, G_TYPE_UINT); + signals[SIGNAL_DISCONNECT] = g_signal_new ("disconnect", + G_TYPE_FROM_CLASS (gobject_class), G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, + G_STRUCT_OFFSET (GstSctpEncClass, disconnect), NULL, NULL, + g_cclosure_marshal_generic, G_TYPE_BOOLEAN, 0); + + signals[SIGNAL_RECONNECT] = g_signal_new ("reconnect", + G_TYPE_FROM_CLASS (gobject_class), G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, + G_STRUCT_OFFSET (GstSctpEncClass, reconnect), NULL, NULL, + g_cclosure_marshal_generic, G_TYPE_NONE, 0); + klass->on_get_stream_bytes_sent = GST_DEBUG_FUNCPTR (on_get_stream_bytes_sent); + klass->disconnect = GST_DEBUG_FUNCPTR (disconnect); + klass->reconnect = GST_DEBUG_FUNCPTR (reconnect); gst_element_class_set_static_metadata (element_class, "SCTP Encoder", @@ -259,10 +290,12 @@ data_queue_full_cb (GstDataQueue * queue, gpointer user_data) static void gst_sctp_enc_init (GstSctpEnc * self) { + g_mutex_init (GST_SCTP_ENC_GET_ASSOC_MUTEX (self)); + + self->sctp_association = NULL; self->sctp_association_id = DEFAULT_GST_SCTP_ASSOCIATION_ID; self->remote_sctp_port = DEFAULT_REMOTE_SCTP_PORT; - self->sctp_association = NULL; self->outbound_sctp_packet_queue = gst_data_queue_new (data_queue_check_full_cb, data_queue_full_cb, data_queue_empty_cb, NULL); @@ -270,20 +303,51 @@ gst_sctp_enc_init (GstSctpEnc * self) self->src_pad = gst_pad_new_from_static_template (&src_template, "src"); gst_pad_set_event_function (self->src_pad, GST_DEBUG_FUNCPTR ((GstPadEventFunction) gst_sctp_enc_src_event)); + gst_pad_set_activatemode_function (self->src_pad, + GST_DEBUG_FUNCPTR (gst_sctp_enc_src_activate_mode)); gst_element_add_pad (GST_ELEMENT (self), self->src_pad); g_queue_init (&self->pending_pads); self->src_ret = GST_FLOW_FLUSHING; } +static void +remove_sinkpad (const GValue * item, gpointer user_data) +{ + GstSctpEncPad *sctpenc_pad = g_value_get_object (item); + GstSctpEnc *self = user_data; + + gst_sctp_enc_release_pad (GST_ELEMENT (self), GST_PAD (sctpenc_pad)); +} + +static void +gst_sctp_enc_dispose (GObject * object) +{ + GstSctpEnc *self = GST_SCTP_ENC_CAST (object); + + /* remove all sinkpads */ + GstIterator *it; + it = gst_element_iterate_sink_pads (GST_ELEMENT (self)); + while (gst_iterator_foreach (it, remove_sinkpad, self) == GST_ITERATOR_RESYNC) + gst_iterator_resync (it); + gst_iterator_free (it); + g_queue_clear (&self->pending_pads); + + gst_pad_set_active (self->src_pad, FALSE); + + G_OBJECT_CLASS (parent_class)->dispose (object); +} + static void gst_sctp_enc_finalize (GObject * object) { - GstSctpEnc *self = GST_SCTP_ENC (object); + GstSctpEnc *self = GST_SCTP_ENC_CAST (object); g_queue_clear (&self->pending_pads); gst_object_unref (self->outbound_sctp_packet_queue); + g_mutex_clear (GST_SCTP_ENC_GET_ASSOC_MUTEX (self)); + G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -291,7 +355,7 @@ static void gst_sctp_enc_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { - GstSctpEnc *self = GST_SCTP_ENC (object); + GstSctpEnc *self = GST_SCTP_ENC_CAST (object); switch (prop_id) { case PROP_GST_SCTP_ASSOCIATION_ID: @@ -299,9 +363,30 @@ gst_sctp_enc_set_property (GObject * object, guint prop_id, break; case PROP_REMOTE_SCTP_PORT: self->remote_sctp_port = g_value_get_uint (value); + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + if (self->sctp_association) { + g_object_set (self->sctp_association, "remote-port", + self->remote_sctp_port, NULL); + } + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); break; case PROP_USE_SOCK_STREAM: self->use_sock_stream = g_value_get_boolean (value); + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + if (self->sctp_association) { + g_object_set (self->sctp_association, "use-sock-stream", + self->use_sock_stream, NULL); + } + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); + break; + case PROP_AGGRESSIVE_HEARTBEAT: + self->aggressive_heartbeat = g_value_get_boolean (value); + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + if (self->sctp_association) { + g_object_set (self->sctp_association, "aggressive-heartbeat", + self->aggressive_heartbeat, NULL); + } + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (self, prop_id, pspec); @@ -313,7 +398,7 @@ static void gst_sctp_enc_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { - GstSctpEnc *self = GST_SCTP_ENC (object); + GstSctpEnc *self = GST_SCTP_ENC_CAST (object); switch (prop_id) { case PROP_GST_SCTP_ASSOCIATION_ID: @@ -325,6 +410,9 @@ gst_sctp_enc_get_property (GObject * object, guint prop_id, GValue * value, case PROP_USE_SOCK_STREAM: g_value_set_boolean (value, self->use_sock_stream); break; + case PROP_AGGRESSIVE_HEARTBEAT: + g_value_set_boolean (value, self->aggressive_heartbeat); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (self, prop_id, pspec); break; @@ -362,61 +450,58 @@ flush_sinkpads (GstSctpEnc * self, gboolean state) static GstStateChangeReturn gst_sctp_enc_change_state (GstElement * element, GstStateChange transition) { - GstSctpEnc *self = GST_SCTP_ENC (element); - GstStateChangeReturn ret = GST_STATE_CHANGE_FAILURE; - gboolean res = TRUE; + GstSctpEnc *self = GST_SCTP_ENC_CAST (element); + GstStateChangeReturn ret; + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); switch (transition) { case GST_STATE_CHANGE_NULL_TO_READY: - break; - case GST_STATE_CHANGE_READY_TO_PAUSED: - self->need_segment = self->need_stream_start_caps = TRUE; - self->src_ret = GST_FLOW_OK; - gst_data_queue_set_flushing (self->outbound_sctp_packet_queue, FALSE); - res = configure_association (self); - break; - case GST_STATE_CHANGE_PLAYING_TO_PAUSED: - break; - case GST_STATE_CHANGE_PAUSED_TO_READY: - stop_srcpad_task (self->src_pad, self); - flush_sinkpads (self, TRUE); - self->src_ret = GST_FLOW_FLUSHING; + if (!configure_association (self)) + ret = GST_STATE_CHANGE_FAILURE; break; case GST_STATE_CHANGE_READY_TO_NULL: + cleanup_association (self); break; default: break; } - if (res) - ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + return ret; +} + +static gboolean +gst_sctp_enc_src_activate_mode (GstPad * pad, GstObject * parent, + GstPadMode mode, gboolean active) +{ + GstSctpEnc *self = GST_SCTP_ENC_CAST (parent); + gboolean result; - switch (transition) { - case GST_STATE_CHANGE_NULL_TO_READY: - break; - case GST_STATE_CHANGE_READY_TO_PAUSED: - gst_pad_start_task (self->src_pad, - (GstTaskFunction) gst_sctp_enc_srcpad_loop, self->src_pad, NULL); - break; - case GST_STATE_CHANGE_PLAYING_TO_PAUSED: - break; - case GST_STATE_CHANGE_PAUSED_TO_READY: - sctpenc_cleanup (self); - break; - case GST_STATE_CHANGE_READY_TO_NULL: - break; - default: - break; + GST_DEBUG_OBJECT (self, "activate mode %d active %d", mode, active); + + if (active) { + self->need_segment = TRUE; + self->need_stream_start_caps = TRUE; + self->src_ret = GST_FLOW_OK; + gst_data_queue_set_flushing (self->outbound_sctp_packet_queue, FALSE); + + result = gst_pad_start_task (pad, + (GstTaskFunction) gst_sctp_enc_srcpad_loop, pad, NULL); + } else { + gst_data_queue_set_flushing (self->outbound_sctp_packet_queue, TRUE); + gst_data_queue_flush (self->outbound_sctp_packet_queue); + result = gst_pad_stop_task (pad); + self->src_ret = GST_FLOW_FLUSHING; } - return ret; + return result; } static GstPad * gst_sctp_enc_request_new_pad (GstElement * element, GstPadTemplate * template, const gchar * new_pad_name, const GstCaps * caps) { - GstSctpEnc *self = GST_SCTP_ENC (element); + GstSctpEnc *self = GST_SCTP_ENC_CAST (element); GstPad *new_pad = NULL; GstSctpEncPad *sctpenc_pad; guint32 stream_id; @@ -424,7 +509,15 @@ gst_sctp_enc_request_new_pad (GstElement * element, GstPadTemplate * template, guint32 new_ppid; gboolean is_new_ppid; + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + if (!self->sctp_association) { + GST_ERROR_OBJECT (self, "Attempt to get pad without a GstSctpAssociation"); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); + return NULL; + } + g_object_get (self->sctp_association, "state", &state, NULL); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); if (state != GST_SCTP_ASSOCIATION_STATE_CONNECTED) { GST_ERROR_OBJECT @@ -461,7 +554,7 @@ gst_sctp_enc_request_new_pad (GstElement * element, GstPadTemplate * template, gst_pad_set_event_function (new_pad, GST_DEBUG_FUNCPTR (gst_sctp_enc_sink_event)); - sctpenc_pad = GST_SCTP_ENC_PAD (new_pad); + sctpenc_pad = GST_SCTP_ENC_PAD_CAST (new_pad); sctpenc_pad->stream_id = stream_id; sctpenc_pad->ppid = DEFAULT_SCTP_PPID; @@ -497,11 +590,12 @@ gst_sctp_enc_request_new_pad (GstElement * element, GstPadTemplate * template, static void gst_sctp_enc_release_pad (GstElement * element, GstPad * pad) { - GstSctpEncPad *sctpenc_pad = GST_SCTP_ENC_PAD (pad); + GstSctpEncPad *sctpenc_pad = GST_SCTP_ENC_PAD_CAST (pad); GstSctpEnc *self; guint stream_id = 0; + GstSctpAssociation *sctp_association = NULL; - self = GST_SCTP_ENC (element); + self = GST_SCTP_ENC_CAST (element); g_mutex_lock (&sctpenc_pad->lock); sctpenc_pad->flushing = TRUE; @@ -511,8 +605,14 @@ gst_sctp_enc_release_pad (GstElement * element, GstPad * pad) stream_id = sctpenc_pad->stream_id; gst_pad_set_active (pad, FALSE); - if (self->sctp_association) - gst_sctp_association_reset_stream (self->sctp_association, stream_id); + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + sctp_association = gst_sctp_association_ref (self->sctp_association); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); + + if (sctp_association) { + gst_sctp_association_reset_stream (sctp_association, stream_id); + gst_sctp_association_unref (sctp_association); + } GST_PAD_STREAM_LOCK (pad); if (gst_object_has_as_parent (GST_OBJECT (pad), GST_OBJECT (element))) @@ -523,7 +623,7 @@ gst_sctp_enc_release_pad (GstElement * element, GstPad * pad) static void gst_sctp_enc_srcpad_loop (GstPad * pad) { - GstSctpEnc *self = GST_SCTP_ENC (GST_PAD_PARENT (pad)); + GstSctpEnc *self = GST_SCTP_ENC_CAST (GST_PAD_PARENT (pad)); GstFlowReturn flow_ret; GstDataQueueItem *item; @@ -592,8 +692,8 @@ gst_sctp_enc_srcpad_loop (GstPad * pad) static GstFlowReturn gst_sctp_enc_sink_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) { - GstSctpEnc *self = GST_SCTP_ENC (parent); - GstSctpEncPad *sctpenc_pad = GST_SCTP_ENC_PAD (pad); + GstSctpEnc *self = GST_SCTP_ENC_CAST (parent); + GstSctpEncPad *sctpenc_pad = GST_SCTP_ENC_PAD_CAST (pad); GstSctpEncPad *sctpenc_pad_next = NULL; GstMapInfo map; guint32 ppid; @@ -679,13 +779,24 @@ gst_sctp_enc_sink_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) while (!sctpenc_pad->flushing) { guint32 bytes_sent; + GstSctpAssociation *sctp_association = NULL; g_mutex_unlock (&sctpenc_pad->lock); - flow_ret = - gst_sctp_association_send_data (self->sctp_association, data, - length, sctpenc_pad->stream_id, ppid, ordered, pr, pr_param, - &bytes_sent); + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + sctp_association = gst_sctp_association_ref (self->sctp_association); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); + + if (sctp_association) { + flow_ret = + gst_sctp_association_send_data (sctp_association, data, + length, sctpenc_pad->stream_id, ppid, ordered, pr, pr_param, + &bytes_sent); + gst_sctp_association_unref (sctp_association); + } else { + GST_ERROR_OBJECT (self, "No GstSctpAssociation"); + flow_ret = GST_FLOW_ERROR; + } g_mutex_lock (&sctpenc_pad->lock); if (flow_ret != GST_FLOW_OK) { @@ -740,8 +851,8 @@ gst_sctp_enc_sink_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) static gboolean gst_sctp_enc_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) { - GstSctpEnc *self = GST_SCTP_ENC (parent); - GstSctpEncPad *sctpenc_pad = GST_SCTP_ENC_PAD (pad); + GstSctpEnc *self = GST_SCTP_ENC_CAST (parent); + GstSctpEncPad *sctpenc_pad = GST_SCTP_ENC_PAD_CAST (pad); gboolean ret, is_new_ppid; guint32 new_ppid; @@ -796,7 +907,7 @@ gst_sctp_enc_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) static gboolean gst_sctp_enc_src_event (GstPad * pad, GstObject * parent, GstEvent * event) { - GstSctpEnc *self = GST_SCTP_ENC (parent); + GstSctpEnc *self = GST_SCTP_ENC_CAST (parent); gboolean ret; switch (GST_EVENT_TYPE (event)) { @@ -818,8 +929,6 @@ gst_sctp_enc_src_event (GstPad * pad, GstObject * parent, GstEvent * event) GST_OBJECT_LOCK (self); self->src_ret = GST_FLOW_OK; GST_OBJECT_UNLOCK (self); - gst_pad_start_task (self->src_pad, - (GstTaskFunction) gst_sctp_enc_srcpad_loop, self->src_pad, NULL); ret = gst_pad_event_default (pad, parent, event); break; @@ -831,49 +940,101 @@ gst_sctp_enc_src_event (GstPad * pad, GstObject * parent, GstEvent * event) return ret; } +static gboolean +disconnect (GstSctpEnc * self) +{ + gint state; + GstSctpAssociation *sctp_association = NULL; + + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + sctp_association = gst_sctp_association_ref (self->sctp_association); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); + + if (!sctp_association) { + GST_ERROR_OBJECT (self, "No GstSctpAssociation"); + return FALSE; + } + + g_object_get (sctp_association, "state", &state, NULL); + if (state != GST_SCTP_ASSOCIATION_STATE_CONNECTED) { + GST_WARNING_OBJECT (self, "Cannot disconnect in non-CONNECTED state."); + gst_sctp_association_unref (sctp_association); + return FALSE; + } + + gst_sctp_association_disconnect (sctp_association); + gst_sctp_association_unref (sctp_association); + + return TRUE; +} + +static void +reconnect (GstSctpEnc * self) +{ + gint state; + GstSctpAssociation *sctp_association = NULL; + + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + sctp_association = gst_sctp_association_ref (self->sctp_association); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); + + if (!sctp_association) { + GST_ERROR_OBJECT (self, "No GstSctpAssociation"); + return; + } + + g_object_get (sctp_association, "state", &state, NULL); + if (state != GST_SCTP_ASSOCIATION_STATE_DISCONNECTED) { + GST_WARNING_OBJECT (self, "Cannot reconnect in non-DISCONNECTED state."); + gst_sctp_association_unref (sctp_association); + return; + } + + gst_sctp_association_start (sctp_association); + gst_sctp_association_unref (sctp_association); +} + static gboolean configure_association (GstSctpEnc * self) { gint state; + GstSctpAssociationEncoderCtx ctx; + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); self->sctp_association = gst_sctp_association_get (self->sctp_association_id); - g_object_get (self->sctp_association, "state", &state, NULL); if (state != GST_SCTP_ASSOCIATION_STATE_NEW) { GST_WARNING_OBJECT (self, "Could not configure SCTP association. Association already in use!"); - g_object_unref (self->sctp_association); + gst_sctp_association_unref (self->sctp_association); self->sctp_association = NULL; - goto error; + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); + return FALSE; } - self->signal_handler_state_changed = - g_signal_connect_object (self->sctp_association, "notify::state", - G_CALLBACK (on_sctp_association_state_changed), self, 0); + g_object_set (self->sctp_association, "remote-port", self->remote_sctp_port, + "use-sock-stream", self->use_sock_stream, "aggressive-heartbeat", + self->aggressive_heartbeat, NULL); - g_object_bind_property (self, "remote-sctp-port", self->sctp_association, - "remote-port", G_BINDING_SYNC_CREATE); + ctx.element = self; + ctx.state_change_cb = on_sctp_association_state_changed; + ctx.packet_out_cb = on_sctp_packet_out; + gst_sctp_association_set_encoder_ctx (self->sctp_association, &ctx); - g_object_bind_property (self, "use-sock-stream", self->sctp_association, - "use-sock-stream", G_BINDING_SYNC_CREATE); - - gst_sctp_association_set_on_packet_out (self->sctp_association, - on_sctp_packet_out, gst_object_ref (self), gst_object_unref); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); return TRUE; -error: - return FALSE; } static void -on_sctp_association_state_changed (GstSctpAssociation * sctp_association, - GParamSpec * pspec, GstSctpEnc * self) +on_sctp_association_state_changed (GstSctpAssociation * + sctp_association, GstSctpAssociationState state, gpointer user_data) { - gint state; - - g_object_get (sctp_association, "state", &state, NULL); + GstSctpEnc *self = (GstSctpEnc *) user_data; + /* we demand to have a valid encoder here */ + g_assert (self); GST_DEBUG_OBJECT (self, "Association state changed to %d", state); switch (state) { @@ -889,6 +1050,7 @@ on_sctp_association_state_changed (GstSctpAssociation * sctp_association, TRUE); break; case GST_SCTP_ASSOCIATION_STATE_DISCONNECTING: + break; case GST_SCTP_ASSOCIATION_STATE_DISCONNECTED: g_signal_emit (self, signals[SIGNAL_SCTP_ASSOCIATION_ESTABLISHED], 0, FALSE); @@ -952,41 +1114,22 @@ on_sctp_packet_out (GstSctpAssociation * _association, const guint8 * buf, } static void -stop_srcpad_task (GstPad * pad, GstSctpEnc * self) -{ - gst_data_queue_set_flushing (self->outbound_sctp_packet_queue, TRUE); - gst_data_queue_flush (self->outbound_sctp_packet_queue); - gst_pad_stop_task (pad); -} - -static void -remove_sinkpad (const GValue * item, gpointer user_data) -{ - GstSctpEncPad *sctpenc_pad = g_value_get_object (item); - GstSctpEnc *self = user_data; - - gst_sctp_enc_release_pad (GST_ELEMENT (self), GST_PAD (sctpenc_pad)); -} - -static void -sctpenc_cleanup (GstSctpEnc * self) +cleanup_association (GstSctpEnc * self) { - GstIterator *it; + GstSctpAssociationEncoderCtx ctx; - gst_sctp_association_set_on_packet_out (self->sctp_association, NULL, NULL, - NULL); + GST_SCTP_ENC_ASSOC_MUTEX_LOCK (self); + if (!self->sctp_association) { + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); + return; + } - g_signal_handler_disconnect (self->sctp_association, - self->signal_handler_state_changed); + memset (&ctx, 0, sizeof (GstSctpAssociationEncoderCtx)); + gst_sctp_association_set_encoder_ctx (self->sctp_association, &ctx); gst_sctp_association_force_close (self->sctp_association); - g_object_unref (self->sctp_association); + gst_sctp_association_unref (self->sctp_association); self->sctp_association = NULL; - - it = gst_element_iterate_sink_pads (GST_ELEMENT (self)); - while (gst_iterator_foreach (it, remove_sinkpad, self) == GST_ITERATOR_RESYNC) - gst_iterator_resync (it); - gst_iterator_free (it); - g_queue_clear (&self->pending_pads); + GST_SCTP_ENC_ASSOC_MUTEX_UNLOCK (self); } static void @@ -1052,7 +1195,7 @@ on_get_stream_bytes_sent (GstSctpEnc * self, guint stream_id) return 0; } - sctpenc_pad = GST_SCTP_ENC_PAD (pad); + sctpenc_pad = GST_SCTP_ENC_PAD_CAST (pad); g_mutex_lock (&sctpenc_pad->lock); bytes_sent = sctpenc_pad->bytes_sent; diff --git a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h index 482473d745e..e8794bc25c9 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h +++ b/subprojects/gst-plugins-bad/ext/sctp/gstsctpenc.h @@ -34,6 +34,7 @@ G_BEGIN_DECLS #define GST_TYPE_SCTP_ENC (gst_sctp_enc_get_type()) #define GST_SCTP_ENC(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_SCTP_ENC, GstSctpEnc)) +#define GST_SCTP_ENC_CAST(obj) (GstSctpEnc*)(obj) #define GST_SCTP_ENC_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_SCTP_ENC, GstSctpEncClass)) #define GST_IS_SCTP_ENC(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_SCTP_ENC)) #define GST_IS_SCTP_ENC_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_SCTP_ENC)) @@ -45,19 +46,20 @@ struct _GstSctpEnc { GstElement element; + GMutex association_mutex; + GstPad *src_pad; GstFlowReturn src_ret; gboolean need_stream_start_caps, need_segment; guint32 sctp_association_id; guint16 remote_sctp_port; gboolean use_sock_stream; + gboolean aggressive_heartbeat; GstSctpAssociation *sctp_association; GstDataQueue *outbound_sctp_packet_queue; GQueue pending_pads; - - gulong signal_handler_state_changed; }; struct _GstSctpEncClass @@ -68,7 +70,8 @@ struct _GstSctpEncClass gboolean established); guint64 (*on_get_stream_bytes_sent) (GstSctpEnc * sctp_enc, guint stream_id); - + gboolean (*disconnect) (GstSctpEnc * sctp_enc); + void (*reconnect) (GstSctpEnc * sctp_enc); }; GType gst_sctp_enc_get_type (void); diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c index 68c05e62f31..8e21186b102 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.c @@ -69,13 +69,6 @@ gst_sctp_association_state_get_type (void) G_DEFINE_TYPE (GstSctpAssociation, gst_sctp_association, G_TYPE_OBJECT); -enum -{ - SIGNAL_STREAM_RESET, - LAST_SIGNAL -}; - - enum { PROP_0, @@ -85,21 +78,20 @@ enum PROP_REMOTE_PORT, PROP_STATE, PROP_USE_SOCK_STREAM, + PROP_AGGRESSIVE_HEARTBEAT, NUM_PROPERTIES }; -static guint signals[LAST_SIGNAL] = { 0 }; - static GParamSpec *properties[NUM_PROPERTIES]; -#define DEFAULT_NUMBER_OF_SCTP_STREAMS 1024 +#define DEFAULT_NUMBER_OF_SCTP_STREAMS 65535 #define DEFAULT_LOCAL_SCTP_PORT 0 #define DEFAULT_REMOTE_SCTP_PORT 0 -static GHashTable *associations = NULL; G_LOCK_DEFINE_STATIC (associations_lock); -static guint32 number_of_associations = 0; +static GHashTable *associations_by_id = NULL; +static GHashTable *ids_by_association = NULL; /* Interface implementations */ static void gst_sctp_association_finalize (GObject * object); @@ -108,6 +100,8 @@ static void gst_sctp_association_set_property (GObject * object, guint prop_id, static void gst_sctp_association_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); +static void force_close_unlocked (GstSctpAssociation * self, + gboolean change_state); static struct socket *create_sctp_socket (GstSctpAssociation * gst_sctp_association); static struct sockaddr_conn get_sctp_socket_address (GstSctpAssociation * @@ -127,9 +121,9 @@ static void handle_stream_reset_event (GstSctpAssociation * self, static void handle_message (GstSctpAssociation * self, guint8 * data, guint32 datalen, guint16 stream_id, guint32 ppid); -static void maybe_set_state_to_ready (GstSctpAssociation * self); -static gboolean gst_sctp_association_change_state (GstSctpAssociation * self, - GstSctpAssociationState new_state, gboolean lock); +static void maybe_set_state_to_ready_unlocked (GstSctpAssociation * self); +static void gst_sctp_association_change_state_unlocked (GstSctpAssociation * + self, GstSctpAssociationState new_state); static void gst_sctp_association_class_init (GstSctpAssociationClass * klass) @@ -142,11 +136,6 @@ gst_sctp_association_class_init (GstSctpAssociationClass * klass) gobject_class->set_property = gst_sctp_association_set_property; gobject_class->get_property = gst_sctp_association_get_property; - signals[SIGNAL_STREAM_RESET] = - g_signal_new ("stream-reset", G_OBJECT_CLASS_TYPE (klass), - G_SIGNAL_RUN_FIRST, G_STRUCT_OFFSET (GstSctpAssociationClass, - on_sctp_stream_reset), NULL, NULL, NULL, G_TYPE_NONE, 1, G_TYPE_UINT); - properties[PROP_ASSOCIATION_ID] = g_param_spec_uint ("association-id", "The SCTP association-id", "The SCTP association-id.", 0, G_MAXUSHORT, DEFAULT_LOCAL_SCTP_PORT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); @@ -171,6 +160,11 @@ gst_sctp_association_class_init (GstSctpAssociationClass * klass) "When TRUE the partial reliability parameters of the channel is ignored.", FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + properties[PROP_AGGRESSIVE_HEARTBEAT] = + g_param_spec_boolean ("aggressive-heartbeat", "Aggressive heartbeat", + "When set to TRUE, set the heartbeat interval to 10ms and the assoc " + "rtx max to 1.", FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + g_object_class_install_properties (gobject_class, NUM_PROPERTIES, properties); } @@ -191,52 +185,13 @@ gst_usrsctp_debug (const gchar * format, ...) static void gst_sctp_association_init (GstSctpAssociation * self) { - /* No need to lock mutex here as long as the function is only called from gst_sctp_association_get */ - if (number_of_associations == 0) { -#if defined(SCTP_DEBUG) && !defined(GST_DISABLE_GST_DEBUG) - usrsctp_init (0, sctp_packet_out, gst_usrsctp_debug); -#else - usrsctp_init (0, sctp_packet_out, NULL); -#endif - - /* Explicit Congestion Notification */ - usrsctp_sysctl_set_sctp_ecn_enable (0); - - /* Do not send ABORTs in response to INITs (1). - * Do not send ABORTs for received Out of the Blue packets (2). - */ - usrsctp_sysctl_set_sctp_blackhole (2); - - /* Enable interleaving messages for different streams (incoming) - * See: https://tools.ietf.org/html/rfc6458#section-8.1.20 - */ - usrsctp_sysctl_set_sctp_default_frag_interleave (2); - - usrsctp_sysctl_set_sctp_nr_outgoing_streams_default - (DEFAULT_NUMBER_OF_SCTP_STREAMS); - -#if defined(SCTP_DEBUG) && !defined(GST_DISABLE_GST_DEBUG) - if (USRSCTP_GST_DEBUG_LEVEL <= GST_LEVEL_MAX - && USRSCTP_GST_DEBUG_LEVEL <= _gst_debug_min - && USRSCTP_GST_DEBUG_LEVEL <= - gst_debug_category_get_threshold (gst_sctp_debug_category)) { - usrsctp_sysctl_set_sctp_debug_on (SCTP_DEBUG_ALL); - } -#endif - } - number_of_associations++; - self->local_port = DEFAULT_LOCAL_SCTP_PORT; self->remote_port = DEFAULT_REMOTE_SCTP_PORT; self->sctp_ass_sock = NULL; - - g_mutex_init (&self->association_mutex); - self->state = GST_SCTP_ASSOCIATION_STATE_NEW; - self->use_sock_stream = TRUE; - usrsctp_register_address ((void *) self); + g_mutex_init (&self->association_mutex); } static void @@ -244,16 +199,15 @@ gst_sctp_association_finalize (GObject * object) { GstSctpAssociation *self = GST_SCTP_ASSOCIATION (object); - G_LOCK (associations_lock); + /* no need to hold the association_lock, it is held under + gst_sctp_association_unref */ + g_hash_table_remove (associations_by_id, + GUINT_TO_POINTER (self->association_id)); - g_hash_table_remove (associations, GUINT_TO_POINTER (self->association_id)); + /* demand we are no longer registered */ + g_assert (!g_hash_table_contains (ids_by_association, self)); - usrsctp_deregister_address ((void *) self); - number_of_associations--; - if (number_of_associations == 0) { - usrsctp_finish (); - } - G_UNLOCK (associations_lock); + g_mutex_clear (&self->association_mutex); G_OBJECT_CLASS (gst_sctp_association_parent_class)->finalize (object); } @@ -290,14 +244,18 @@ gst_sctp_association_set_property (GObject * object, guint prop_id, case PROP_USE_SOCK_STREAM: self->use_sock_stream = g_value_get_boolean (value); break; + case PROP_AGGRESSIVE_HEARTBEAT: + self->aggressive_heartbeat = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (self, prop_id, pspec); break; } - g_mutex_unlock (&self->association_mutex); if (prop_id == PROP_LOCAL_PORT || prop_id == PROP_REMOTE_PORT) - maybe_set_state_to_ready (self); + maybe_set_state_to_ready_unlocked (self); + + g_mutex_unlock (&self->association_mutex); return; @@ -306,23 +264,16 @@ gst_sctp_association_set_property (GObject * object, guint prop_id, } static void -maybe_set_state_to_ready (GstSctpAssociation * self) +maybe_set_state_to_ready_unlocked (GstSctpAssociation * self) { - gboolean signal_ready_state = FALSE; - - g_mutex_lock (&self->association_mutex); - if ((self->state == GST_SCTP_ASSOCIATION_STATE_NEW) && - (self->local_port != 0 && self->remote_port != 0) - && (self->packet_out_cb != NULL) && (self->packet_received_cb != NULL)) { - signal_ready_state = - gst_sctp_association_change_state (self, - GST_SCTP_ASSOCIATION_STATE_READY, FALSE); + if ((self->state == GST_SCTP_ASSOCIATION_STATE_NEW) + && (self->local_port != 0 && self->remote_port != 0) + && (self->encoder_ctx.packet_out_cb != NULL) + && (self->decoder_ctx.packet_received_cb != NULL) + && (self->encoder_ctx.state_change_cb != NULL)) { + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_READY); } - g_mutex_unlock (&self->association_mutex); - - if (signal_ready_state) - g_object_notify_by_pspec (G_OBJECT (self), properties[PROP_STATE]); - } static void @@ -347,12 +298,111 @@ gst_sctp_association_get_property (GObject * object, guint prop_id, case PROP_USE_SOCK_STREAM: g_value_set_boolean (value, self->use_sock_stream); break; + case PROP_AGGRESSIVE_HEARTBEAT: + g_value_set_boolean (value, self->aggressive_heartbeat); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (self, prop_id, pspec); break; } } +static void +gst_sctp_association_usrsctp_init (void) +{ +#if defined(SCTP_DEBUG) && !defined(GST_DISABLE_GST_DEBUG) + usrsctp_init (0, sctp_packet_out, gst_usrsctp_debug); +#else + usrsctp_init (0, sctp_packet_out, NULL); +#endif + + /* Explicit Congestion Notification */ + usrsctp_sysctl_set_sctp_ecn_enable (0); + + /* Do not send ABORTs in response to INITs (1). + * Do not send ABORTs for received Out of the Blue packets (2). + */ + usrsctp_sysctl_set_sctp_blackhole (2); + + /* Enable interleaving messages for different streams (incoming) + * See: https://tools.ietf.org/html/rfc6458#section-8.1.20 + */ + usrsctp_sysctl_set_sctp_default_frag_interleave (2); + + usrsctp_sysctl_set_sctp_nr_outgoing_streams_default + (DEFAULT_NUMBER_OF_SCTP_STREAMS); + +#if defined(SCTP_DEBUG) && !defined(GST_DISABLE_GST_DEBUG) + if (USRSCTP_GST_DEBUG_LEVEL <= GST_LEVEL_MAX + && USRSCTP_GST_DEBUG_LEVEL <= _gst_debug_min + && USRSCTP_GST_DEBUG_LEVEL <= + gst_debug_category_get_threshold (gst_sctp_debug_category)) { + usrsctp_sysctl_set_sctp_debug_on (SCTP_DEBUG_ALL); + } +#endif +} + +static void +gst_sctp_association_usrsctp_deinit (void) +{ + /* usrsctp_finish could fail, so retry for 5 seconds */ + gint ret; + for (gint i = 0; i < 50; ++i) { + ret = usrsctp_finish (); + if (ret == 0) { + GST_DEBUG ("usrsctp_finish() succeed"); + break; + } + + GST_DEBUG ("usrsctp_finish() failed and returned %d", ret); + g_usleep (100 * G_TIME_SPAN_MILLISECOND); + } + + if (ret != 0) { + GST_WARNING ("usrsctp_finish() failed and returned %d", ret); + } +} + +/* +* Helper register/deregister functions to workaround bug sctplab/usrsctp#405 +* +* The sctp socket can outlive the association, so we need to protect ourselves +* against being called with an invalid reference of GstSctpAssociation. +* To do so, only register/deregister when we create/close the socket in a +* thread-safe way. +* +*/ +static void +gst_sctp_association_register (GstSctpAssociation * self) +{ + G_LOCK (associations_lock); + + /* demand we are not registering twice */ + g_assert (!g_hash_table_contains (ids_by_association, self)); + + g_hash_table_insert (ids_by_association, self, + GUINT_TO_POINTER (self->association_id)); + + G_UNLOCK (associations_lock); + + usrsctp_register_address ((void *) self); +} + +static void +gst_sctp_association_deregister (GstSctpAssociation * self) +{ + usrsctp_deregister_address ((void *) self); + + G_LOCK (associations_lock); + + /* demand we are not deregistering twice */ + g_assert (g_hash_table_contains (ids_by_association, self)); + + g_hash_table_remove (ids_by_association, self); + + G_UNLOCK (associations_lock); +} + /* Public functions */ GstSctpAssociation * @@ -366,18 +416,25 @@ gst_sctp_association_get (guint32 association_id) GST_DEBUG_CATEGORY_INIT (gst_sctp_debug_category, "sctplib", 0, "debug category for messages from usrsctp"); - if (!associations) { - associations = + if (!associations_by_id) { + g_assert (ids_by_association == NULL); + + associations_by_id = g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, NULL); + ids_by_association = + g_hash_table_new_full (g_direct_hash, g_direct_equal, NULL, NULL); + + gst_sctp_association_usrsctp_init (); } association = - g_hash_table_lookup (associations, GUINT_TO_POINTER (association_id)); + g_hash_table_lookup (associations_by_id, + GUINT_TO_POINTER (association_id)); if (!association) { association = g_object_new (GST_SCTP_TYPE_ASSOCIATION, "association-id", association_id, NULL); - g_hash_table_insert (associations, GUINT_TO_POINTER (association_id), + g_hash_table_insert (associations_by_id, GUINT_TO_POINTER (association_id), association); } else { g_object_ref (association); @@ -386,10 +443,42 @@ gst_sctp_association_get (guint32 association_id) return association; } -gboolean -gst_sctp_association_start (GstSctpAssociation * self) +GstSctpAssociation * +gst_sctp_association_ref (GstSctpAssociation * self) +{ + GstSctpAssociation *ref = NULL; + G_LOCK (associations_lock); + if (self) + ref = g_object_ref (self); + G_UNLOCK (associations_lock); + return ref; +} + +void +gst_sctp_association_unref (GstSctpAssociation * self) +{ + G_LOCK (associations_lock); + g_object_unref (self); + + if (g_hash_table_size (associations_by_id) == 0) { + /* demand all association have ben deregistered */ + g_assert (g_hash_table_size (ids_by_association) == 0); + + g_hash_table_destroy (associations_by_id); + g_hash_table_destroy (ids_by_association); + associations_by_id = NULL; + ids_by_association = NULL; + + gst_sctp_association_usrsctp_deinit (); + } + G_UNLOCK (associations_lock); +} + +static gboolean +gst_sctp_association_start_unlocked (GstSctpAssociation * self) { - if (self->state != GST_SCTP_ASSOCIATION_STATE_READY) { + if (self->state != GST_SCTP_ASSOCIATION_STATE_READY && + self->state != GST_SCTP_ASSOCIATION_STATE_DISCONNECTED) { GST_WARNING_OBJECT (self, "SCTP association is in wrong state and cannot be started"); goto configure_required; @@ -400,64 +489,82 @@ gst_sctp_association_start (GstSctpAssociation * self) /* TODO: Support both server and client role */ if (!client_role_connect (self)) { - gst_sctp_association_change_state (self, GST_SCTP_ASSOCIATION_STATE_ERROR, - TRUE); + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_ERROR); goto error; } - gst_sctp_association_change_state (self, - GST_SCTP_ASSOCIATION_STATE_CONNECTING, TRUE); + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_CONNECTING); return TRUE; error: - gst_sctp_association_change_state (self, GST_SCTP_ASSOCIATION_STATE_ERROR, - TRUE); + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_ERROR); return FALSE; configure_required: return FALSE; } +gboolean +gst_sctp_association_start (GstSctpAssociation * self) +{ + g_mutex_lock (&self->association_mutex); + gboolean ret = gst_sctp_association_start_unlocked (self); + g_mutex_unlock (&self->association_mutex); + return ret; +} + void -gst_sctp_association_set_on_packet_out (GstSctpAssociation * self, - GstSctpAssociationPacketOutCb packet_out_cb, gpointer user_data, - GDestroyNotify destroy_notify) +gst_sctp_association_set_encoder_ctx (GstSctpAssociation * self, + GstSctpAssociationEncoderCtx * ctx) { g_return_if_fail (GST_SCTP_IS_ASSOCIATION (self)); g_mutex_lock (&self->association_mutex); - if (self->packet_out_destroy_notify) - self->packet_out_destroy_notify (self->packet_out_user_data); - self->packet_out_cb = packet_out_cb; - self->packet_out_user_data = user_data; - self->packet_out_destroy_notify = destroy_notify; - g_mutex_unlock (&self->association_mutex); - maybe_set_state_to_ready (self); + if (self->encoder_ctx.element) + gst_object_unref (self->encoder_ctx.element); + + g_assert (ctx); + self->encoder_ctx = *ctx; + + if (ctx->element) + self->encoder_ctx.element = gst_object_ref (ctx->element); + + maybe_set_state_to_ready_unlocked (self); + g_mutex_unlock (&self->association_mutex); } void -gst_sctp_association_set_on_packet_received (GstSctpAssociation * self, - GstSctpAssociationPacketReceivedCb packet_received_cb, gpointer user_data, - GDestroyNotify destroy_notify) +gst_sctp_association_set_decoder_ctx (GstSctpAssociation * self, + GstSctpAssociationDecoderCtx * ctx) { g_return_if_fail (GST_SCTP_IS_ASSOCIATION (self)); g_mutex_lock (&self->association_mutex); - if (self->packet_received_destroy_notify) - self->packet_received_destroy_notify (self->packet_received_user_data); - self->packet_received_cb = packet_received_cb; - self->packet_received_user_data = user_data; - self->packet_received_destroy_notify = destroy_notify; - g_mutex_unlock (&self->association_mutex); - maybe_set_state_to_ready (self); + if (self->decoder_ctx.element) + gst_object_unref (self->decoder_ctx.element); + + g_assert (ctx); + self->decoder_ctx = *ctx; + + if (ctx->element) + self->decoder_ctx.element = gst_object_ref (ctx->element); + + maybe_set_state_to_ready_unlocked (self); + g_mutex_unlock (&self->association_mutex); } void gst_sctp_association_incoming_packet (GstSctpAssociation * self, const guint8 * buf, guint32 length) { - usrsctp_conninput ((void *) self, (const void *) buf, (size_t) length, 0); + g_mutex_lock (&self->association_mutex); + if (self->sctp_ass_sock) + usrsctp_conninput ((void *) self, (const void *) buf, (size_t) length, 0); + g_mutex_unlock (&self->association_mutex); } GstFlowReturn @@ -475,7 +582,7 @@ gst_sctp_association_send_data (GstSctpAssociation * self, const guint8 * buf, if (self->state != GST_SCTP_ASSOCIATION_STATE_CONNECTED) { if (self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTED || self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { - GST_ERROR_OBJECT (self, "Disconnected"); + GST_INFO_OBJECT (self, "Disconnected"); flow_ret = GST_FLOW_EOS; g_mutex_unlock (&self->association_mutex); goto end; @@ -487,7 +594,6 @@ gst_sctp_association_send_data (GstSctpAssociation * self, const guint8 * buf, } } remote_addr = get_sctp_socket_address (self, self->remote_port); - g_mutex_unlock (&self->association_mutex); /* TODO: We probably want to split too large chunks into multiple packets * and only set the SCTP_EOR flag on the last one. Firefox is using 0x4000 @@ -516,6 +622,9 @@ gst_sctp_association_send_data (GstSctpAssociation * self, const guint8 * buf, usrsctp_sendv (self->sctp_ass_sock, buf, length, (struct sockaddr *) &remote_addr, 1, (void *) &spa, (socklen_t) sizeof (struct sctp_sendv_spa), SCTP_SENDV_SPA, 0); + + g_mutex_unlock (&self->association_mutex); + if (bytes_sent < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK) { bytes_sent = 0; @@ -544,30 +653,79 @@ gst_sctp_association_reset_stream (GstSctpAssociation * self, guint16 stream_id) struct sctp_reset_streams *srs; socklen_t length; + g_mutex_lock (&self->association_mutex); + + if (self->state != GST_SCTP_ASSOCIATION_STATE_CONNECTED) { + /* only allow resets on connected streams */ + g_mutex_unlock (&self->association_mutex); + return; + } + length = (socklen_t) (sizeof (struct sctp_reset_streams) + sizeof (guint16)); srs = (struct sctp_reset_streams *) g_malloc0 (length); - srs->srs_assoc_id = SCTP_ALL_ASSOC; srs->srs_flags = SCTP_STREAM_RESET_OUTGOING; srs->srs_number_streams = 1; srs->srs_stream_list[0] = stream_id; + srs->srs_assoc_id = self->sctp_assoc_id; - usrsctp_setsockopt (self->sctp_ass_sock, IPPROTO_SCTP, SCTP_RESET_STREAMS, - srs, length); + if (usrsctp_setsockopt (self->sctp_ass_sock, IPPROTO_SCTP, + SCTP_RESET_STREAMS, srs, length) < 0) { + GST_WARNING_OBJECT (self, "Resetting stream id=%u failed", stream_id); + } + + g_mutex_unlock (&self->association_mutex); g_free (srs); } void gst_sctp_association_force_close (GstSctpAssociation * self) +{ + g_mutex_lock (&self->association_mutex); + force_close_unlocked (self, TRUE); + g_mutex_unlock (&self->association_mutex); +} + +static void +force_close_unlocked (GstSctpAssociation * self, gboolean change_state) { if (self->sctp_ass_sock) { - struct socket *s = self->sctp_ass_sock; + usrsctp_close (self->sctp_ass_sock); + gst_sctp_association_deregister (self); self->sctp_ass_sock = NULL; - usrsctp_close (s); } - gst_sctp_association_change_state (self, - GST_SCTP_ASSOCIATION_STATE_DISCONNECTED, TRUE); + self->sctp_assoc_id = 0; + + if (change_state) { + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_DISCONNECTED); + } +} + +static void +gst_sctp_association_disconnect_unlocked (GstSctpAssociation * self) +{ + if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_DISCONNECTING); + } + + /* Fall through to ensure the transition to disconnected occurs */ + if (self->state == GST_SCTP_ASSOCIATION_STATE_DISCONNECTING) { + force_close_unlocked (self, FALSE); + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_DISCONNECTED); + GST_INFO_OBJECT (self, "SCTP association disconnected!"); + } +} + +void +gst_sctp_association_disconnect (GstSctpAssociation * self) +{ + g_mutex_lock (&self->association_mutex); + gst_sctp_association_disconnect_unlocked (self); + g_mutex_unlock (&self->association_mutex); } static struct socket * @@ -681,6 +839,8 @@ create_sctp_socket (GstSctpAssociation * self) } } + gst_sctp_association_register (self); + return sock; error: if (sock) @@ -713,19 +873,34 @@ client_role_connect (GstSctpAssociation * self) socklen_t opt_len; gint ret; - g_mutex_lock (&self->association_mutex); local_addr = get_sctp_socket_address (self, self->local_port); remote_addr = get_sctp_socket_address (self, self->remote_port); - g_mutex_unlock (&self->association_mutex); - ret = - usrsctp_bind (self->sctp_ass_sock, (struct sockaddr *) &local_addr, - sizeof (struct sockaddr_conn)); - if (ret < 0) { - GST_ERROR_OBJECT (self, "usrsctp_bind() error: (%u) %s", errno, - g_strerror (errno)); - goto error; - } + /* After an SCTP association is reported as disconnected, there is + * a window of time before the underlying SCTP stack cleans up. + * If a client-initiated reconnect request occurs during this window + * then we will attempt to bind using the same address information + * which will fail with EADDRINUSE. Handle this by retrying whenever + * a bind fails in this way. + */ + size_t retry_count = 1; + do { + ret = + usrsctp_bind (self->sctp_ass_sock, (struct sockaddr *) &local_addr, + sizeof (struct sockaddr_conn)); + if (ret < 0) { + if (errno != EADDRINUSE || retry_count == 10) { + GST_ERROR_OBJECT (self, "usrsctp_bind() error: (%u) %s", errno, + g_strerror (errno)); + goto error; + } else { + GST_WARNING_OBJECT (self, "usrsctp_bind() error: (%u) %s", errno, + g_strerror (errno)); + } + g_usleep (G_USEC_PER_SEC / 100); + retry_count++; + } + } while (ret < 0); ret = usrsctp_connect (self->sctp_ass_sock, (struct sockaddr *) &remote_addr, @@ -770,17 +945,31 @@ client_role_connect (GstSctpAssociation * self) return FALSE; } +static gboolean +association_is_valid (GstSctpAssociation * self) +{ + gboolean valid = FALSE; + + G_LOCK (associations_lock); + + if (ids_by_association != NULL) + valid = g_hash_table_contains (ids_by_association, self); + + G_UNLOCK (associations_lock); + + return valid; +} + static int sctp_packet_out (void *addr, void *buffer, size_t length, guint8 tos, guint8 set_df) { GstSctpAssociation *self = GST_SCTP_ASSOCIATION (addr); - g_mutex_lock (&self->association_mutex); - if (self->packet_out_cb) { - self->packet_out_cb (self, buffer, length, self->packet_out_user_data); + if (association_is_valid (self) && self->encoder_ctx.packet_out_cb) { + self->encoder_ctx.packet_out_cb (self, buffer, length, + self->encoder_ctx.element); } - g_mutex_unlock (&self->association_mutex); return 0; } @@ -791,9 +980,19 @@ receive_cb (struct socket *sock, union sctp_sockstore addr, void *data, { GstSctpAssociation *self = GST_SCTP_ASSOCIATION (ulp_info); + if (!association_is_valid (self)) { + return 1; + } + + + /* If we acquire the lock here it means this is the SCTP timer thread, if + the thread has already acquired the lock, its coming from + gst_sctp_association_incoming_packet() */ + gboolean acquired_lock = g_mutex_trylock (&self->association_mutex); + if (!data) { - /* Not sure if this can happend. */ - GST_WARNING_OBJECT (self, "Received empty data buffer"); + /* This is a notification that socket shutdown is complete */ + GST_INFO_OBJECT (self, "Received shutdown complete notification"); } else { if (flags & MSG_NOTIFICATION) { handle_notification (self, (const union sctp_notification *) data, @@ -810,6 +1009,9 @@ receive_cb (struct socket *sock, union sctp_sockstore addr, void *data, } } + if (acquired_lock) + g_mutex_unlock (&self->association_mutex); + return 1; } @@ -836,8 +1038,8 @@ handle_notification (GstSctpAssociation * self, break; case SCTP_SHUTDOWN_EVENT: GST_DEBUG_OBJECT (self, "Event: SCTP_SHUTDOWN_EVENT"); - gst_sctp_association_change_state (self, - GST_SCTP_ASSOCIATION_STATE_DISCONNECTING, TRUE); + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_DISCONNECTING); break; case SCTP_ADAPTATION_INDICATION: GST_DEBUG_OBJECT (self, "Event: SCTP_ADAPTATION_INDICATION"); @@ -874,49 +1076,127 @@ handle_notification (GstSctpAssociation * self, } static void -handle_association_changed (GstSctpAssociation * self, +_apply_aggressive_heartbeat_unlocked (GstSctpAssociation * self) +{ + struct sctp_assocparams assoc_params; + struct sctp_paddrparams peer_addr_params; + struct sockaddr_conn addr; + + if (!self->aggressive_heartbeat) + return; + + memset (&assoc_params, 0, sizeof (assoc_params)); + assoc_params.sasoc_assoc_id = self->sctp_assoc_id; + assoc_params.sasoc_asocmaxrxt = 1; + if (usrsctp_setsockopt (self->sctp_ass_sock, IPPROTO_SCTP, + SCTP_ASSOCINFO, &assoc_params, sizeof (assoc_params))) { + GST_WARNING_OBJECT (self, "Could not set SCTP_ASSOCINFO"); + } + + addr = get_sctp_socket_address (self, self->remote_port); + memset (&peer_addr_params, 0, sizeof (peer_addr_params)); + memcpy (&peer_addr_params.spp_address, &addr, sizeof (addr)); + peer_addr_params.spp_flags = SPP_HB_ENABLE; + peer_addr_params.spp_hbinterval = 10; + if (usrsctp_setsockopt (self->sctp_ass_sock, IPPROTO_SCTP, + SCTP_PEER_ADDR_PARAMS, &peer_addr_params, + sizeof (peer_addr_params))) { + GST_WARNING_OBJECT (self, "Could not set SCTP_PEER_ADDR_PARAMS"); + } +} + +static void +handle_sctp_comm_up (GstSctpAssociation * self, + const struct sctp_assoc_change *sac) +{ + GST_INFO_OBJECT (self, "SCTP_COMM_UP"); + if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTING) { + self->sctp_assoc_id = sac->sac_assoc_id; + _apply_aggressive_heartbeat_unlocked (self); + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_CONNECTED); + GST_INFO_OBJECT (self, "SCTP association connected!"); + } else if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { + GST_INFO_OBJECT (self, "SCTP association already open"); + } else { + GST_WARNING_OBJECT (self, "SCTP association in unexpected state: %d", + self->state); + } +} + +static void +handle_sctp_comm_lost_or_shutdown (GstSctpAssociation * self, const struct sctp_assoc_change *sac) { - gboolean change_state = FALSE; - GstSctpAssociationState new_state; + GST_INFO_OBJECT (self, "SCTP event %s received", + sac->sac_state == SCTP_COMM_LOST ? + "SCTP_COMM_LOST" : "SCTP_SHUTDOWN_COMP"); + + gst_sctp_association_disconnect_unlocked (self); +} + +static void +gst_sctp_association_notify_restart (GstSctpAssociation * self) +{ + GstSctpAssociationRestartCb restart_cb; + gpointer user_data; + + restart_cb = self->decoder_ctx.restart_cb; + user_data = self->decoder_ctx.element; + if (user_data) + gst_object_ref (user_data); + + if (restart_cb) + restart_cb (self, user_data); + + if (user_data) + gst_object_unref (user_data); +} + +static void +gst_sctp_association_notify_stream_reset (GstSctpAssociation * self, + guint16 stream_id) +{ + GstSctpAssociationStreamResetCb stream_reset_cb; + gpointer user_data; + + stream_reset_cb = self->decoder_ctx.stream_reset_cb; + user_data = self->decoder_ctx.element; + if (user_data) + gst_object_ref (user_data); + + if (stream_reset_cb) + stream_reset_cb (self, stream_id, user_data); + if (user_data) + gst_object_unref (user_data); +} + +static void +handle_association_changed (GstSctpAssociation * self, + const struct sctp_assoc_change *sac) +{ switch (sac->sac_state) { case SCTP_COMM_UP: - GST_DEBUG_OBJECT (self, "SCTP_COMM_UP"); - g_mutex_lock (&self->association_mutex); - if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTING) { - change_state = TRUE; - new_state = GST_SCTP_ASSOCIATION_STATE_CONNECTED; - GST_DEBUG_OBJECT (self, "SCTP association connected!"); - } else if (self->state == GST_SCTP_ASSOCIATION_STATE_CONNECTED) { - GST_FIXME_OBJECT (self, "SCTP association already open"); - } else { - GST_WARNING_OBJECT (self, "SCTP association in unexpected state"); - } - g_mutex_unlock (&self->association_mutex); + handle_sctp_comm_up (self, sac); break; case SCTP_COMM_LOST: - GST_WARNING_OBJECT (self, "SCTP event SCTP_COMM_LOST received"); - change_state = TRUE; - new_state = GST_SCTP_ASSOCIATION_STATE_ERROR; + handle_sctp_comm_lost_or_shutdown (self, sac); break; case SCTP_RESTART: - GST_DEBUG_OBJECT (self, "SCTP event SCTP_RESTART received"); + GST_INFO_OBJECT (self, "SCTP event SCTP_RESTART received"); + gst_sctp_association_notify_restart (self); break; case SCTP_SHUTDOWN_COMP: - GST_DEBUG_OBJECT (self, "SCTP event SCTP_SHUTDOWN_COMP received"); - change_state = TRUE; - new_state = GST_SCTP_ASSOCIATION_STATE_DISCONNECTED; + /* Occurs if in TCP mode when the far end sends SHUTDOWN */ + handle_sctp_comm_lost_or_shutdown (self, sac); break; case SCTP_CANT_STR_ASSOC: GST_WARNING_OBJECT (self, "SCTP event SCTP_CANT_STR_ASSOC received"); - change_state = TRUE; - new_state = GST_SCTP_ASSOCIATION_STATE_ERROR; + gst_sctp_association_change_state_unlocked (self, + GST_SCTP_ASSOCIATION_STATE_ERROR); break; } - - if (change_state) - gst_sctp_association_change_state (self, new_state, TRUE); } static void @@ -930,7 +1210,7 @@ handle_stream_reset_event (GstSctpAssociation * self, sizeof (struct sctp_stream_reset_event)) / sizeof (uint16_t); for (i = 0; i < n; i++) { if (sr->strreset_flags & SCTP_STREAM_RESET_INCOMING_SSN) { - g_signal_emit (self, signals[SIGNAL_STREAM_RESET], 0, + gst_sctp_association_notify_stream_reset (self, sr->strreset_stream_list[i]); } } @@ -941,11 +1221,10 @@ static void handle_message (GstSctpAssociation * self, guint8 * data, guint32 datalen, guint16 stream_id, guint32 ppid) { - g_mutex_lock (&self->association_mutex); - if (self->packet_received_cb) { + if (self->decoder_ctx.packet_received_cb) { /* It's the callbacks job to free the data correctly */ - self->packet_received_cb (self, data, datalen, stream_id, ppid, - self->packet_received_user_data); + self->decoder_ctx.packet_received_cb (self, data, datalen, stream_id, ppid, + self->decoder_ctx.element); } else { /* We use this instead of a bare `free()` so that we use the `free` from * the C runtime that usrsctp was built with. This makes a difference on @@ -953,30 +1232,42 @@ handle_message (GstSctpAssociation * self, guint8 * data, guint32 datalen, * CRTs. */ usrsctp_freedumpbuffer ((gchar *) data); } - g_mutex_unlock (&self->association_mutex); } -/* Returns TRUE if lock==FALSE and notification is needed later. - * Takes the mutex shortly if lock==TRUE! */ -static gboolean -gst_sctp_association_change_state (GstSctpAssociation * self, - GstSctpAssociationState new_state, gboolean lock) +static void +gst_sctp_association_change_state_unlocked (GstSctpAssociation * self, + GstSctpAssociationState new_state) { - if (lock) - g_mutex_lock (&self->association_mutex); + gboolean notify = FALSE; + GstSctpAssociationStateChangeCb callback = self->encoder_ctx.state_change_cb; + gpointer encoder = self->encoder_ctx.element; + if (self->state != new_state && self->state != GST_SCTP_ASSOCIATION_STATE_ERROR) { self->state = new_state; - if (lock) { - g_mutex_unlock (&self->association_mutex); - g_object_notify_by_pspec (G_OBJECT (self), properties[PROP_STATE]); - return FALSE; - } else { - return TRUE; - } - } else { - if (lock) - g_mutex_unlock (&self->association_mutex); - return FALSE; + notify = TRUE; } + + /* return immediately if we don't have to notify */ + if (!notify) + return; + + /* hold a ref on the association and the encoder, so we make sure they + outlives the callback execution */ + gst_sctp_association_ref (self); + if (encoder) + gst_object_ref (encoder); + + /* release the association mutex, so other calls can be done to the + association */ + g_mutex_unlock (&self->association_mutex); + + if (callback) + callback (self, new_state, encoder); + + g_mutex_lock (&self->association_mutex); + + if (encoder) + gst_object_unref (encoder); + gst_sctp_association_unref (self); } diff --git a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h index 81332904408..d618f248384 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h +++ b/subprojects/gst-plugins-bad/ext/sctp/sctpassociation.h @@ -28,8 +28,6 @@ #include #include -#define INET -#define INET6 #include G_BEGIN_DECLS @@ -44,6 +42,9 @@ G_BEGIN_DECLS typedef struct _GstSctpAssociation GstSctpAssociation; typedef struct _GstSctpAssociationClass GstSctpAssociationClass; +typedef struct _GstSctpAssociationEncoderCtx GstSctpAssociationEncoderCtx; +typedef struct _GstSctpAssociationDecoderCtx GstSctpAssociationDecoderCtx; + typedef enum { GST_SCTP_ASSOCIATION_STATE_NEW, @@ -63,11 +64,35 @@ typedef enum GST_SCTP_ASSOCIATION_PARTIAL_RELIABILITY_RTX = 0x0003 } GstSctpAssociationPartialReliability; +typedef void (*GstSctpAssociationPacketOutCb) (GstSctpAssociation * + sctp_association, const guint8 * data, gsize length, gpointer user_data); +typedef void (*GstSctpAssociationStateChangeCb) (GstSctpAssociation * + sctp_association, GstSctpAssociationState state, gpointer user_data); + +struct _GstSctpAssociationEncoderCtx +{ + gpointer element; + GstSctpAssociationStateChangeCb state_change_cb; + GstSctpAssociationPacketOutCb packet_out_cb; +}; + typedef void (*GstSctpAssociationPacketReceivedCb) (GstSctpAssociation * sctp_association, guint8 * data, gsize length, guint16 stream_id, guint ppid, gpointer user_data); -typedef void (*GstSctpAssociationPacketOutCb) (GstSctpAssociation * - sctp_association, const guint8 * data, gsize length, gpointer user_data); + +typedef void (*GstSctpAssociationStreamResetCb)(GstSctpAssociation * + sctp_association, guint16 stream_id, gpointer user_data); + +typedef void (*GstSctpAssociationRestartCb)(GstSctpAssociation * + sctp_association, gpointer user_data); + +struct _GstSctpAssociationDecoderCtx +{ + gpointer element; + GstSctpAssociationPacketReceivedCb packet_received_cb; + GstSctpAssociationStreamResetCb stream_reset_cb; + GstSctpAssociationRestartCb restart_cb; +}; struct _GstSctpAssociation { @@ -77,38 +102,38 @@ struct _GstSctpAssociation guint16 local_port; guint16 remote_port; gboolean use_sock_stream; + gboolean aggressive_heartbeat; struct socket *sctp_ass_sock; GMutex association_mutex; GstSctpAssociationState state; - GstSctpAssociationPacketReceivedCb packet_received_cb; - gpointer packet_received_user_data; - GDestroyNotify packet_received_destroy_notify; + guint32 sctp_assoc_id; - GstSctpAssociationPacketOutCb packet_out_cb; - gpointer packet_out_user_data; - GDestroyNotify packet_out_destroy_notify; + GstSctpAssociationEncoderCtx encoder_ctx; + GstSctpAssociationDecoderCtx decoder_ctx; }; struct _GstSctpAssociationClass { GObjectClass parent_class; - - void (*on_sctp_stream_reset) (GstSctpAssociation * sctp_association, - guint16 stream_id); }; GType gst_sctp_association_get_type (void); GstSctpAssociation *gst_sctp_association_get (guint32 association_id); +GstSctpAssociation *gst_sctp_association_ref (GstSctpAssociation * self); +void gst_sctp_association_unref (GstSctpAssociation * self); + gboolean gst_sctp_association_start (GstSctpAssociation * self); -void gst_sctp_association_set_on_packet_out (GstSctpAssociation * self, - GstSctpAssociationPacketOutCb packet_out_cb, gpointer user_data, GDestroyNotify destroy_notify); -void gst_sctp_association_set_on_packet_received (GstSctpAssociation * self, - GstSctpAssociationPacketReceivedCb packet_received_cb, gpointer user_data, GDestroyNotify destroy_notify); + +void gst_sctp_association_set_encoder_ctx (GstSctpAssociation * self, + GstSctpAssociationEncoderCtx * ctx); +void gst_sctp_association_set_decoder_ctx (GstSctpAssociation * self, + GstSctpAssociationDecoderCtx * ctx); + void gst_sctp_association_incoming_packet (GstSctpAssociation * self, const guint8 * buf, guint32 length); GstFlowReturn gst_sctp_association_send_data (GstSctpAssociation * self, @@ -118,6 +143,7 @@ GstFlowReturn gst_sctp_association_send_data (GstSctpAssociation * self, void gst_sctp_association_reset_stream (GstSctpAssociation * self, guint16 stream_id); void gst_sctp_association_force_close (GstSctpAssociation * self); +void gst_sctp_association_disconnect (GstSctpAssociation * self); G_END_DECLS diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/meson.build b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/meson.build index 6a1ab845da9..e1f57ee7bad 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/meson.build +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/meson.build @@ -143,7 +143,7 @@ if have_sconn_len endif # Options -if false +if true compile_args += ['-DINVARIANTS'] endif if not gst_debug_disabled diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_output.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_output.c index a494b2c0ae4..ccd7a55f444 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_output.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_output.c @@ -7385,6 +7385,10 @@ sctp_sendall(struct sctp_inpcb *inp, struct uio *uio, struct mbuf *m, int ret; struct sctp_copy_all *ca; + if (!uio) { + return (EFAULT); + } + if (inp->sctp_flags & SCTP_PCB_FLAGS_SND_ITERATOR_UP) { /* There is another. */ return (EBUSY); diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c index 6fd3c5e2e1a..4c41273f187 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_pcb.c @@ -901,6 +901,15 @@ sctp_del_addr_from_vrf(uint32_t vrf_id, struct sockaddr *addr, sctp_free_ifa(sctp_ifap); return; } +#if defined(__Userspace__) + else if (sctp_ifap->address.sa.sa_family == AF_CONN) { + /* Clean up immediately */ + SCTPDBG(SCTP_DEBUG_PCB4, "Immediately deleting AF_CONN\n"); + sctp_free_ifa(sctp_ifap); + SCTP_ZONE_FREE(SCTP_BASE_INFO(ipi_zone_laddr), wi); + return; + } +#endif SCTP_INCR_LADDR_COUNT(); memset(wi, 0, sizeof(*wi)); (void)SCTP_GETTIME_TIMEVAL(&wi->start_time); @@ -6715,7 +6724,13 @@ sctp_pcb_init(void) (void)pthread_cond_init(&sctp_it_ctl.iterator_wakeup, NULL); #endif #endif +#if defined(SCTP_ITERATOR) sctp_startup_iterator(); +#else + SCTP_ITERATOR_LOCK_INIT(); + SCTP_IPI_ITERATOR_WQ_INIT(); + TAILQ_INIT(&sctp_it_ctl.iteratorhead); +#endif #if defined(__FreeBSD__) && !defined(__Userspace__) #if defined(SCTP_MCORE_INPUT) && defined(SMP) @@ -6770,6 +6785,7 @@ sctp_pcb_finish(void) return; } SCTP_BASE_VAR(sctp_pcb_initialized) = 0; +#if defined(SCTP_ITERATOR) #if !(defined(__FreeBSD__) && !defined(__Userspace__)) /* Notify the iterator to exit. */ SCTP_IPI_ITERATOR_WQ_LOCK(); @@ -6810,11 +6826,13 @@ sctp_pcb_finish(void) CloseHandle(sctp_it_ctl.thread_proc); sctp_it_ctl.thread_proc = NULL; #else + pthread_cancel(sctp_it_ctl.thread_proc); pthread_join(sctp_it_ctl.thread_proc, NULL); sctp_it_ctl.thread_proc = 0; #endif } #endif +#endif #if defined(SCTP_PROCESS_LEVEL_LOCKS) #if defined(_WIN32) DeleteConditionVariable(&sctp_it_ctl.iterator_wakeup); @@ -8062,6 +8080,10 @@ sctp_initiate_iterator(inp_func inpf, { struct sctp_iterator *it = NULL; +#if !defined(SCTP_ITERATOR) + return (-1); +#endif + if (af == NULL) { return (-1); } diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_usrreq.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_usrreq.c index bf66b362006..a66c73d085b 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_usrreq.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/netinet/sctp_usrreq.c @@ -87,7 +87,7 @@ sctp_init(void) u_long sb_max_adj; #else - init_random(); + init_usrsctp_random(); #endif /* Initialize and modify the sysctled variables */ sctp_init_sysctls(); diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.c index dc9d98a3920..f88cf280021 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.c @@ -66,7 +66,7 @@ userland_mutex_t atomic_mtx; */ #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION void -init_random(void) +init_usrsctp_random(void) { return; } @@ -79,7 +79,7 @@ read_random(void *buf, int count) } #elif defined(__FreeBSD__) || defined(__DragonFly__) || defined(__OpenBSD__) || defined(__OpenBSD__) || defined(__APPLE__) void -init_random(void) +init_usrsctp_random(void) { return; } @@ -94,7 +94,7 @@ read_random(void *buf, int count) } #else void -init_random(void) +init_usrsctp_random(void) { struct timeval now; unsigned int seed; diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.h b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.h index 0c2ec9db235..3414d8683b1 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.h +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_environment.h @@ -67,7 +67,7 @@ extern int nmbclusters; #define max(a,b) (((a)>(b))?(a):(b)) #endif -void init_random(void); +void init_usrsctp_random(void); int read_random(void *, int); /* errno's may differ per OS. errno.h now included in sctp_os_userspace.h */ diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_mbuf.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_mbuf.c index 9e566e30259..948ce7e06c8 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_mbuf.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_mbuf.c @@ -122,6 +122,7 @@ m_get(int how, short type) /* Mbuf master zone, zone_mbuf, has already been * created in mbuf_initialize() */ mret = SCTP_ZONE_GET(zone_mbuf, struct mbuf); + memset(mret, 0, sizeof (struct mbuf)); #if defined(SCTP_SIMPLE_ALLOCATOR) mb_ctor_mbuf(mret, &mbuf_mb_args, 0); #endif @@ -162,6 +163,7 @@ m_gethdr(int how, short type) mbuf_mb_args.type = type; #endif mret = SCTP_ZONE_GET(zone_mbuf, struct mbuf); + memset(mret, 0, sizeof (struct mbuf)); #if defined(SCTP_SIMPLE_ALLOCATOR) mb_ctor_mbuf(mret, &mbuf_mb_args, 0); #endif diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_recv_thread.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_recv_thread.c index 3c1657b4a66..3579834c817 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_recv_thread.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_recv_thread.c @@ -213,7 +213,10 @@ recv_function_route(void *arg) len = recvmsg(SCTP_BASE_VAR(userspace_route), &msg, 0); - if (len < 0) { + if (len == 0) + break; + + if (len < 0) { if (errno == EAGAIN || errno == EINTR) { continue; } else { @@ -341,6 +344,9 @@ recv_function_raw(void *arg) msg.msg_control = NULL; msg.msg_controllen = 0; ncounter = n = recvmsg(SCTP_BASE_VAR(userspace_rawsctp), &msg, 0); + if (n == 0) + break; + if (n < 0) { if (errno == EAGAIN || errno == EINTR) { continue; @@ -533,6 +539,9 @@ recv_function_raw6(void *arg) msg.msg_flags = 0; ncounter = n = recvmsg(SCTP_BASE_VAR(userspace_rawsctp6), &msg, 0); + if (n == 0) + break; + if (n < 0) { if (errno == EAGAIN || errno == EINTR) { continue; @@ -702,13 +711,18 @@ recv_function_udp(void *arg) msg.msg_flags = 0; ncounter = n = recvmsg(SCTP_BASE_VAR(userspace_udpsctp), &msg, 0); - if (n < 0) { + + if (n == 0) + break; + + if (n < 0) { if (errno == EAGAIN || errno == EINTR) { continue; } else { break; } } + #else nResult = WSAIoctl(SCTP_BASE_VAR(userspace_udpsctp), SIO_GET_EXTENSION_FUNCTION_POINTER, &WSARecvMsg_GUID, sizeof WSARecvMsg_GUID, @@ -904,6 +918,8 @@ recv_function_udp6(void *arg) msg.msg_flags = 0; ncounter = n = recvmsg(SCTP_BASE_VAR(userspace_udpsctp6), &msg, 0); + if (n == 0) + break; if (n < 0) { if (errno == EAGAIN || errno == EINTR) { continue; @@ -1452,6 +1468,19 @@ recv_thread_init(void) #endif } +#if !defined(_WIN32) +static void +close_socket(int *sock) +{ + if (*sock != -1) { + int socktmp = *sock; + *sock = -1; + shutdown (socktmp, SHUT_RDWR); + close(socktmp); + } +} +#endif /* !defined(_WIN32) */ + void recv_thread_destroy(void) { @@ -1471,7 +1500,7 @@ recv_thread_destroy(void) WaitForSingleObject(SCTP_BASE_VAR(recvthreadraw), INFINITE); CloseHandle(SCTP_BASE_VAR(recvthreadraw)); #else - close(SCTP_BASE_VAR(userspace_rawsctp)); + close_socket(&SCTP_BASE_VAR(userspace_rawsctp)); SCTP_BASE_VAR(userspace_rawsctp) = -1; pthread_join(SCTP_BASE_VAR(recvthreadraw), NULL); #endif @@ -1483,7 +1512,7 @@ recv_thread_destroy(void) WaitForSingleObject(SCTP_BASE_VAR(recvthreadudp), INFINITE); CloseHandle(SCTP_BASE_VAR(recvthreadudp)); #else - close(SCTP_BASE_VAR(userspace_udpsctp)); + close_socket(&SCTP_BASE_VAR(userspace_udpsctp)); SCTP_BASE_VAR(userspace_udpsctp) = -1; pthread_join(SCTP_BASE_VAR(recvthreadudp), NULL); #endif @@ -1497,7 +1526,7 @@ recv_thread_destroy(void) WaitForSingleObject(SCTP_BASE_VAR(recvthreadraw6), INFINITE); CloseHandle(SCTP_BASE_VAR(recvthreadraw6)); #else - close(SCTP_BASE_VAR(userspace_rawsctp6)); + close_socket(&SCTP_BASE_VAR(userspace_rawsctp6)); SCTP_BASE_VAR(userspace_rawsctp6) = -1; pthread_join(SCTP_BASE_VAR(recvthreadraw6), NULL); #endif @@ -1509,7 +1538,7 @@ recv_thread_destroy(void) WaitForSingleObject(SCTP_BASE_VAR(recvthreadudp6), INFINITE); CloseHandle(SCTP_BASE_VAR(recvthreadudp6)); #else - close(SCTP_BASE_VAR(userspace_udpsctp6)); + close_socket(&SCTP_BASE_VAR(userspace_udpsctp6)); SCTP_BASE_VAR(userspace_udpsctp6) = -1; pthread_join(SCTP_BASE_VAR(recvthreadudp6), NULL); #endif diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socket.c b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socket.c index b72b05ebc30..ce9d922cb1b 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socket.c +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socket.c @@ -1880,6 +1880,8 @@ soconnect(struct socket *so, struct sockaddr *nam) */ if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) && (error = sodisconnect(so))) { error = EISCONN; + } else if (nam == NULL){ + return EINVAL; } else { /* * Prevent accumulated error from previous connection from diff --git a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socketvar.h b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socketvar.h index 6b79b5908c0..6e7870260c5 100644 --- a/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socketvar.h +++ b/subprojects/gst-plugins-bad/ext/sctp/usrsctp/usrsctplib/user_socketvar.h @@ -403,6 +403,7 @@ void sofree(struct socket *so); #define soref(so) do { \ SOCK_LOCK_ASSERT(so); \ + KASSERT((so)->so_count > 0, ("soref")); \ ++(so)->so_count; \ } while (0) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/meson.build b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/meson.build index 2c0a48cf180..6ffa3942560 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/meson.build +++ b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/meson.build @@ -31,8 +31,26 @@ pkgconfig.generate(libgstsctp, description : 'SCTP helper functions', ) +sctp_gen_sources = [] +if build_gir + sctp_gir = gnome.generate_gir(libgstsctp, + sources : sctp_sources + sctp_headers, + namespace : 'GstSctp', + nsversion : api_version, + identifier_prefix : 'Gst', + symbol_prefix : 'gst', + export_packages : 'gstreamer-sctp-1.0', + includes : ['Gst-1.0'], + install : true, + extra_args : gir_init_section + ['-DGST_USE_UNSTABLE_API'], + dependencies : [gstbase_dep] + ) + sctp_gen_sources += sctp_gir +endif + gstsctp_dep = declare_dependency(link_with : libgstsctp, include_directories : [libsinc], + sources : sctp_gen_sources, dependencies : [gstbase_dep]) meson.override_dependency(pkg_name, gstsctp_dep) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.c b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.c index 11def65238d..4a21b74ca01 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.c +++ b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.c @@ -79,6 +79,15 @@ gst_sctp_receive_meta_transform (GstBuffer * transbuf, GstMeta * meta, return TRUE; } +/** + * gst_sctp_buffer_add_receive_meta: + * @buffer: a #GstBuffer + * @ppid: sctp ppid + * + * Attaches @ppid as metadata in a #GstSctpReceiveMeta to @buffer. + * + * Returns: (transfer none): a #GstSctpReceiveMeta connected to @buffer + */ GstSctpReceiveMeta * gst_sctp_buffer_add_receive_meta (GstBuffer * buffer, guint32 ppid) { @@ -91,3 +100,19 @@ gst_sctp_buffer_add_receive_meta (GstBuffer * buffer, guint32 ppid) gst_sctp_receive_meta->ppid = ppid; return gst_sctp_receive_meta; } + +/** + * gst_sctp_buffer_get_receive_meta: + * @buffer: a #GstBuffer + * + * Find the #GstSctpReceiveMeta on @buffer. + * + * Returns: (transfer none): the #GstSctpReceiveMeta or %NULL when there + * is no such metadata on @buffer. + */ +GstSctpReceiveMeta * +gst_sctp_buffer_get_receive_meta (GstBuffer * buffer) +{ + return (GstSctpReceiveMeta *) + gst_buffer_get_meta(buffer, GST_SCTP_RECEIVE_META_API_TYPE); +} diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.h b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.h index 5a508370c2f..e8f2f488319 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.h +++ b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpreceivemeta.h @@ -49,8 +49,8 @@ const GstMetaInfo *gst_sctp_receive_meta_get_info (void); GST_SCTP_API GstSctpReceiveMeta *gst_sctp_buffer_add_receive_meta (GstBuffer * buffer, guint32 ppid); - -#define gst_sctp_buffer_get_receive_meta(b) ((GstSctpReceiveMeta *)gst_buffer_get_meta((b), GST_SCTP_RECEIVE_META_API_TYPE)) +GST_SCTP_API +GstSctpReceiveMeta *gst_sctp_buffer_get_receive_meta (GstBuffer * buffer); G_END_DECLS diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.c b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.c index e29cd7d9273..9b18c9c3ef0 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.c +++ b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.c @@ -84,6 +84,18 @@ gst_sctp_send_meta_transform (GstBuffer * transbuf, GstMeta * meta, return TRUE; } +/** + * gst_sctp_buffer_add_send_meta: + * @buffer: a #GstBuffer + * @ppid: sctp ppid + * @ordered: whether buffer should be sent ordered + * @pr: partial reliability mode + * @pr_param: partial reliability parameter + * + * Attaches metadata in a #GstSctpSendMeta to @buffer. + * + * Returns: (transfer none): a #GstSctpSendMeta connected to @buffer + */ GstSctpSendMeta * gst_sctp_buffer_add_send_meta (GstBuffer * buffer, guint32 ppid, gboolean ordered, GstSctpSendMetaPartiallyReliability pr, guint32 pr_param) @@ -100,3 +112,19 @@ gst_sctp_buffer_add_send_meta (GstBuffer * buffer, guint32 ppid, gst_sctp_send_meta->pr_param = pr_param; return gst_sctp_send_meta; } + +/** + * gst_sctp_buffer_get_send_meta: + * @buffer: a #GstBuffer + * + * Find the #GstSctpSendMeta on @buffer. + * + * Returns: (transfer none): the #GstSctpSendMeta or %NULL when there + * is no such metadata on @buffer. + */ +GstSctpSendMeta * +gst_sctp_buffer_get_send_meta (GstBuffer * buffer) +{ + return (GstSctpSendMeta *) + gst_buffer_get_meta(buffer, GST_SCTP_SEND_META_API_TYPE); +} diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.h b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.h index b1860e52f22..3b117a47d9e 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.h +++ b/subprojects/gst-plugins-bad/gst-libs/gst/sctp/sctpsendmeta.h @@ -61,8 +61,8 @@ GST_SCTP_API GstSctpSendMeta *gst_sctp_buffer_add_send_meta (GstBuffer * buffer, guint32 ppid, gboolean ordered, GstSctpSendMetaPartiallyReliability pr, guint32 pr_param); - -#define gst_sctp_buffer_get_send_meta(b) ((GstSctpSendMeta *)gst_buffer_get_meta((b), GST_SCTP_SEND_META_API_TYPE)) +GST_SCTP_API +GstSctpSendMeta *gst_sctp_buffer_get_send_meta (GstBuffer * buffer); G_END_DECLS diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m index 8e4a91e1e2f..6c6237dc50b 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m +++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m @@ -62,6 +62,7 @@ { gpointer internal_win_id; gpointer internal_view; + gpointer external_view; gint preferred_width; gint preferred_height; @@ -80,6 +81,8 @@ static gboolean gst_vulkan_window_cocoa_get_presentation_support (GstVulkanWindo static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, GError ** error); static void gst_vulkan_window_cocoa_close (GstVulkanWindow * window); +static void gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, + guintptr window_handle); static void gst_vulkan_window_cocoa_finalize (GObject * object) @@ -100,6 +103,7 @@ static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, window_class->get_surface = gst_vulkan_window_cocoa_get_surface; window_class->get_presentation_support = gst_vulkan_window_cocoa_get_presentation_support; + window_class->set_window_handle = gst_vulkan_window_cocoa_set_window_handle; } static void @@ -154,9 +158,14 @@ static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, GstVulkanWindowCocoa *window_cocoa = GST_VULKAN_WINDOW_COCOA (window); GstVulkanWindowCocoaPrivate *priv = GET_PRIV (window_cocoa); - if (!priv->visible) - _gst_vk_invoke_on_main ((GstVulkanWindowFunc) _show_window, - gst_object_ref (window), (GDestroyNotify) gst_object_unref); + if (!priv->visible) { + if (priv->external_view) { + gst_vulkan_window_cocoa_set_window_handle (window, (guintptr) priv->external_view); + } else { + _gst_vk_invoke_on_main ((GstVulkanWindowFunc) _show_window, + gst_object_ref (window), (GDestroyNotify) gst_object_unref); + } + } } static void @@ -191,6 +200,10 @@ static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, priv->internal_win_id = (__bridge_retained gpointer)internal_win_id; priv->internal_view = (__bridge_retained gpointer)view; + if (priv->external_view) + gst_vulkan_window_cocoa_set_window_handle (GST_VULKAN_WINDOW (window_cocoa), + (guintptr) priv->external_view); + gst_vulkan_window_cocoa_show (GST_VULKAN_WINDOW (window_cocoa)); } @@ -205,20 +218,84 @@ static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, return TRUE; } -static VkSurfaceKHR -gst_vulkan_window_cocoa_get_surface (GstVulkanWindow * window, GError ** error) +typedef struct +{ + VkResult result; + VkSurfaceKHR surface; + VkMacOSSurfaceCreateInfoMVK * info; + GstVulkanWindow * window; + + GMutex mutex; + GCond cond; + gboolean called; +} GstVulkanCococa_CreateMacOSSurfaceCtx; + +static void +_gst_vulkan_cocoa_wait_for_createmacossurface (GstVulkanCococa_CreateMacOSSurfaceCtx * ctx) { + g_mutex_lock (&ctx->mutex); + while (!ctx->called) + g_cond_wait(&ctx->cond, &ctx->mutex); + g_mutex_unlock (&ctx->mutex); + + g_mutex_clear (&ctx->mutex); + g_cond_clear (&ctx->cond); +} + +static void +_gst_vulkan_cocoa_create_macos_surface_on_main_thread (gpointer data) +{ + GstVulkanCococa_CreateMacOSSurfaceCtx * ctx = data; + GstVulkanWindow *window = ctx->window; GstVulkanWindowCocoa *window_cocoa = GST_VULKAN_WINDOW_COCOA (window); - GstVulkanWindowCocoaPrivate *priv = GET_PRIV (window_cocoa); - VkMacOSSurfaceCreateInfoMVK info = { 0, }; - VkSurfaceKHR ret; + + ctx->result = + window_cocoa->CreateMacOSSurface (window->display->instance->instance, ctx->info, + NULL, &ctx->surface); + + g_mutex_lock (&ctx->mutex); + ctx->called = TRUE; + g_cond_signal (&ctx->cond); + g_mutex_unlock (&ctx->mutex); +} + +static VkResult +_gst_vulkan_cocoa_create_macos_surface (GstVulkanWindow *window, VkSurfaceKHR * surface) +{ VkResult err; + VkMacOSSurfaceCreateInfoMVK info = { 0, }; + + GstVulkanWindowCocoa *window_cocoa = GST_VULKAN_WINDOW_COCOA (window); + GstVulkanWindowCocoaPrivate *priv = GET_PRIV (window_cocoa); + + GstVulkanCococa_CreateMacOSSurfaceCtx ctx; info.sType = VK_STRUCTURE_TYPE_MACOS_SURFACE_CREATE_INFO_MVK; info.pNext = NULL; info.flags = 0; info.pView = priv->internal_view; + memset(&ctx, 0, sizeof(GstVulkanCococa_CreateMacOSSurfaceCtx)); + g_mutex_init (&ctx.mutex); + g_cond_init (&ctx.cond); + ctx.info = &info; + ctx.window = window; + + _gst_vk_invoke_on_main ((GstVulkanWindowFunc) _gst_vulkan_cocoa_create_macos_surface_on_main_thread, &ctx, NULL); + _gst_vulkan_cocoa_wait_for_createmacossurface (&ctx); + + *surface = ctx.surface; + + return ctx.result; +} + +static VkSurfaceKHR +gst_vulkan_window_cocoa_get_surface (GstVulkanWindow * window, GError ** error) +{ + GstVulkanWindowCocoa *window_cocoa = GST_VULKAN_WINDOW_COCOA (window); + VkSurfaceKHR ret; + VkResult err; + if (!window_cocoa->CreateMacOSSurface) window_cocoa->CreateMacOSSurface = gst_vulkan_instance_get_proc_address (window->display->instance, @@ -229,9 +306,7 @@ static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, return VK_NULL_HANDLE; } - err = - window_cocoa->CreateMacOSSurface (window->display->instance->instance, &info, - NULL, &ret); + err = _gst_vulkan_cocoa_create_macos_surface (window, &ret); if (gst_vulkan_error_to_g_error (err, error, "vkCreateMacOSSurfaceMVK") < 0) return VK_NULL_HANDLE; @@ -245,6 +320,57 @@ static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, return TRUE; } +static void +_gst_vulkan_window_cocoa_insert_internal_win (gpointer data) +{ + GstVulkanWindowCocoa *window_cocoa; + GstVulkanWindowCocoaPrivate *priv; + + window_cocoa = (GstVulkanWindowCocoa*) data; + priv = GET_PRIV(window_cocoa); + + GstVulkanNSWindow *internal_win_id = (__bridge GstVulkanNSWindow *)priv->internal_win_id; + NSView *external_view = (__bridge NSView *)priv->external_view; + NSView *view = (__bridge NSView *)priv->internal_view; + + [internal_win_id orderOut:internal_win_id]; + [external_view addSubview: view]; + + [external_view setAutoresizesSubviews: YES]; + [view setFrame: [external_view bounds]]; + [view setAutoresizingMask: NSViewWidthSizable|NSViewHeightSizable]; +} + +static void +gst_vulkan_window_cocoa_set_window_handle (GstVulkanWindow * window, + guintptr handle) +{ + GstVulkanWindowCocoa *window_cocoa; + GstVulkanWindowCocoaPrivate *priv; + + window_cocoa = GST_VULKAN_WINDOW_COCOA (window); + priv = GET_PRIV(window_cocoa); + + if (priv->internal_win_id) { + if (handle) { + priv->external_view = (gpointer)handle; + priv->visible = TRUE; + } else { + /* bring back our internal window */ + priv->external_view = 0; + priv->visible = FALSE; + } + + _gst_vk_invoke_on_main ((GstVulkanWindowFunc) _gst_vulkan_window_cocoa_insert_internal_win, + gst_object_ref (window), (GDestroyNotify) gst_object_unref); + + } else { + /* no internal window yet so delay it to the next drawing */ + priv->external_view = (gpointer)handle; + priv->visible = FALSE; + } +} + static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, GError ** error) { @@ -260,7 +386,7 @@ static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, _close_window (gpointer * data) { GstVulkanWindowCocoa *window_cocoa = GST_VULKAN_WINDOW_COCOA (data); - GstVulkanWindow *window = GST_VULKAN_WINDOW (window_cocoa); + GstVulkanWindow *window = GST_VULKAN_WINDOW (window_cocoa) ; GstVulkanWindowCocoaPrivate *priv = GET_PRIV (window_cocoa); GstVulkanNSWindow *internal_win_id = (__bridge GstVulkanNSWindow *) priv->internal_win_id; @@ -268,9 +394,12 @@ static gboolean gst_vulkan_window_cocoa_open (GstVulkanWindow * window, gst_vulkan_window_cocoa_hide (window); [[internal_win_id contentView] removeFromSuperview]; + [internal_win_id close]; + CFBridgingRelease (priv->internal_win_id); - priv->internal_win_id = NULL; CFBridgingRelease (priv->internal_view); + + priv->internal_win_id = NULL; priv->internal_view = NULL; } diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkwindow.c b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkwindow.c index da2834a2b00..3993b874f40 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkwindow.c +++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkwindow.c @@ -106,18 +106,6 @@ enum static guint gst_vulkan_window_signals[LAST_SIGNAL] = { 0 }; -static gboolean -_accum_logical_and (GSignalInvocationHint * ihint, GValue * return_accu, - const GValue * handler_return, gpointer data) -{ - gboolean val = g_value_get_boolean (handler_return); - gboolean val2 = g_value_get_boolean (return_accu); - - g_value_set_boolean (return_accu, val && val2); - - return TRUE; -} - GQuark gst_vulkan_window_error_quark (void) { @@ -194,7 +182,7 @@ gst_vulkan_window_class_init (GstVulkanWindowClass * klass) gst_vulkan_window_signals[SIGNAL_CLOSE] = g_signal_new ("close", G_TYPE_FROM_CLASS (klass), G_SIGNAL_RUN_LAST, 0, - (GSignalAccumulator) _accum_logical_and, NULL, NULL, G_TYPE_BOOLEAN, 0); + NULL, NULL, NULL, G_TYPE_NONE, 0); gst_vulkan_window_signals[SIGNAL_DRAW] = g_signal_new ("draw", G_TYPE_FROM_CLASS (klass), G_SIGNAL_RUN_LAST, 0, @@ -410,16 +398,14 @@ void gst_vulkan_window_close (GstVulkanWindow * window) { GstVulkanWindowClass *klass; - gboolean to_close; g_return_if_fail (GST_IS_VULKAN_WINDOW (window)); klass = GST_VULKAN_WINDOW_GET_CLASS (window); g_return_if_fail (klass->close != NULL); - g_signal_emit (window, gst_vulkan_window_signals[SIGNAL_CLOSE], 0, &to_close); + g_signal_emit (window, gst_vulkan_window_signals[SIGNAL_CLOSE], 0); - if (to_close) - klass->close (window); + klass->close (window); } /** diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build index 8a1e946d1e8..0a6524f8731 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build +++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/meson.build @@ -88,6 +88,7 @@ has_vulkan_header = false vulkan_dep = dependency('', required: false) vulkan_inc_dir = '' enabled_vulkan_winsys = [] +is_ios_simulator = host_system == 'ios' and meson.get_external_property('needs_exe_wrapper', false) vulkan_conf = configuration_data() vulkan_conf_options = [ @@ -111,7 +112,36 @@ if ['ios', 'darwin'].contains(host_system) # retrieving the metal device from the VkDevice) which is currently waiting # on implementing a proper Metal extension for Vulkan # https://github.com/KhronosGroup/MoltenVK/issues/492 - vulkan_dep = cc.find_library('MoltenVK', required : get_option('vulkan')) + + # 1. Download and install the Vulkan SDK for macOS: + # https://vulkan.lunarg.com/sdk/home#mac + # 2. Set a shell environment variable to point to the SDK: + # `export VK_SDK_PATH="/path/to/VulkanSDK/1.3.231.1"` + vulkan_root = run_command(python3, '-c', 'import os; print(os.environ.get("VK_SDK_PATH"))', check: false).stdout().strip() + + if vulkan_root != '' and vulkan_root != 'None' + molten_vk_root = join_paths(vulkan_root, 'MoltenVK') + + if is_ios_simulator + platform_dir = 'ios-arm64_x86_64-simulator' + elif host_system == 'ios' + platform_dir = 'ios-arm64' + else + platform_dir = 'macos-arm64_x86_64' + endif + + vulkan_lib_dir = join_paths(molten_vk_root, 'MoltenVK.xcframework', platform_dir) + vulkan_inc_dir = join_paths(molten_vk_root, 'include') + vulkan_lib = cc.find_library('MoltenVK', dirs: vulkan_lib_dir, + required : get_option('vulkan')) + has_vulkan_header = cc.has_header('vulkan/vulkan_core.h', + include_directories: include_directories(vulkan_inc_dir)) + vulkan_dep = declare_dependency(include_directories: include_directories(vulkan_inc_dir), + dependencies: vulkan_lib, + link_args: ['-lc++']) + else + subdir_done() + endif elif host_system == 'windows' vulkan_root = run_command(python3, '-c', 'import os; print(os.environ.get("VK_SDK_PATH"))', check: false).stdout().strip() if vulkan_root != '' and vulkan_root != 'None' @@ -141,7 +171,7 @@ else endif endif -if host_system != 'windows' +if not ['ios', 'darwin', 'windows'].contains(host_system) has_vulkan_header = cc.has_header('vulkan/vulkan_core.h') endif @@ -224,41 +254,60 @@ if ['darwin', 'ios'].contains(host_system) vulkan_objc_args += ['-fobjc-arc'] - foundation_dep = dependency('appleframeworks', modules : ['Foundation'], required : get_option('vulkan')) - quartzcore_dep = dependency('appleframeworks', modules : ['QuartzCore'], required : get_option('vulkan')) - corefoundation_dep = dependency('appleframeworks', modules : ['CoreFoundation'], required : get_option('vulkan')) - if foundation_dep.found() and quartzcore_dep.found() and corefoundation_dep.found() - optional_deps += [foundation_dep, corefoundation_dep, quartzcore_dep] - endif + apple_deps = [ + dependency('appleframeworks', modules : ['Foundation'], required : get_option('vulkan')), + dependency('appleframeworks', modules : ['QuartzCore'], required : get_option('vulkan')), + dependency('appleframeworks', modules : ['CoreFoundation'], required : get_option('vulkan')), + dependency('appleframeworks', modules : ['Metal'], required : get_option('vulkan')), + dependency('appleframeworks', modules : ['IOSurface'], required : get_option('vulkan')), + ] + + foreach dep : apple_deps + if dep.found() + optional_deps += [dep] + endif + endforeach endif if host_system == 'darwin' cocoa_dep = dependency('appleframeworks', modules : ['Cocoa'], required : get_option('vulkan')) - if cocoa_dep.found() and cc.has_header('vulkan/vulkan_macos.h', dependencies : vulkan_dep) + if cocoa_dep.found() and cc.has_header('vulkan/vulkan_macos.h', include_directories: include_directories(vulkan_inc_dir)) vulkan_priv_sources += files( 'cocoa/gstvkdisplay_cocoa.m', 'cocoa/gstvkwindow_cocoa.m', ) - optional_deps += [cocoa_dep] vulkan_windowing = true vulkan_conf.set('GST_VULKAN_HAVE_WINDOW_COCOA', 1) enabled_vulkan_winsys += ['cocoa'] + + iokit_dep = dependency('appleframeworks', modules : ['IOKit'], required : get_option('vulkan')) + if cocoa_dep.found() + optional_deps += [iokit_dep] + endif + + optional_deps += [cocoa_dep] endif endif if host_system == 'ios' uikit_dep = dependency('appleframeworks', modules : ['UIKit'], required : get_option('vulkan')) - if uikit_dep.found() and cc.has_header('vulkan/vulkan_ios.h', dependencies : vulkan_dep) + if uikit_dep.found() and cc.has_header('vulkan/vulkan_ios.h', include_directories: include_directories(vulkan_inc_dir)) vulkan_priv_sources += files( 'ios/gstvkdisplay_ios.m', 'ios/gstvkwindow_ios.m', ) - optional_deps += [uikit_dep] vulkan_windowing = true vulkan_conf.set('GST_VULKAN_HAVE_WINDOW_IOS', 1) enabled_vulkan_winsys += ['ios'] + + coregraphics_dep = dependency('appleframeworks', modules : ['CoreGraphics'], required : get_option('vulkan')) + if coregraphics_dep.found() + optional_deps += [coregraphics_dep] + endif + + optional_deps += [uikit_dep] endif endif diff --git a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c index 55853b8fd01..67db0a98c3b 100644 --- a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c +++ b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.c @@ -31,6 +31,7 @@ #include #include #include +#include GST_DEBUG_CATEGORY (netsim_debug); #define GST_CAT_DEFAULT (netsim_debug) @@ -53,6 +54,11 @@ distribution_get_type (void) return static_g_define_type_id; } +#define GST_NET_SIM_LOCK(obj) g_mutex_lock (&obj->mutex) +#define GST_NET_SIM_UNLOCK(obj) g_mutex_unlock (&obj->mutex) +#define GST_NET_SIM_SIGNAL(obj) g_cond_signal (&obj->cond) +#define GST_NET_SIM_WAIT(obj) g_cond_wait (&obj->cond, &obj->mutex) + enum { PROP_0, @@ -65,7 +71,12 @@ enum PROP_DROP_PACKETS, PROP_MAX_KBPS, PROP_MAX_BUCKET_SIZE, + PROP_QUEUE_SIZE, + PROP_MAX_QUEUE_DELAY, PROP_ALLOW_REORDERING, + PROP_REPLACE_DROPPED_WITH_EMPTY, + PROP_THROTTLE_FREQUENCY, + PROP_THROTTLE_DELAY, }; /* these numbers are nothing but wild guesses and don't reflect any reality */ @@ -78,7 +89,12 @@ enum #define DEFAULT_DROP_PACKETS 0 #define DEFAULT_MAX_KBPS -1 #define DEFAULT_MAX_BUCKET_SIZE -1 +#define DEFAULT_QUEUE_SIZE -1 +#define DEFAULT_MAX_QUEUE_DELAY 50 #define DEFAULT_ALLOW_REORDERING TRUE +#define DEFAULT_REPLACE_DROPPED_WITH_EMPTY FALSE +#define DEFAULT_THROTTLE_FREQUENCY 0 +#define DEFAULT_THROTTLE_DELAY 0 static GstStaticPadTemplate gst_net_sim_sink_template = GST_STATIC_PAD_TEMPLATE ("sink", @@ -96,148 +112,101 @@ G_DEFINE_TYPE (GstNetSim, gst_net_sim, GST_TYPE_ELEMENT); GST_ELEMENT_REGISTER_DEFINE (netsim, "netsim", GST_RANK_MARGINAL, GST_TYPE_NET_SIM); -static gboolean -gst_net_sim_source_dispatch (GSource * source, - GSourceFunc callback, gpointer user_data) +typedef struct { - callback (user_data); - return FALSE; + GstBuffer *buf; + guint size_bits; + guint seqnum; + GstClockTime arrival_time; + GstClockTime delay; +} NetSimBuffer; + +static NetSimBuffer * +net_sim_buffer_new (GstBuffer * buf, + guint seqnum, GstClockTime arrival_time, GstClockTime delay) +{ + NetSimBuffer *nsbuf = g_new (NetSimBuffer, 1); + nsbuf->buf = gst_buffer_ref (buf); + nsbuf->size_bits = gst_buffer_get_size (buf) * 8; + nsbuf->seqnum = seqnum; + nsbuf->arrival_time = arrival_time; + nsbuf->delay = delay; + return nsbuf; } -GSourceFuncs gst_net_sim_source_funcs = { - NULL, /* prepare */ - NULL, /* check */ - gst_net_sim_source_dispatch, - NULL /* finalize */ -}; - -static void -gst_net_sim_loop (GstNetSim * netsim) +static GstClockTime +net_sim_buffer_get_push_time (NetSimBuffer * nsbuf) { - GMainLoop *loop; - - GST_TRACE_OBJECT (netsim, "TASK: begin"); - - g_mutex_lock (&netsim->loop_mutex); - loop = g_main_loop_ref (netsim->main_loop); - netsim->running = TRUE; - GST_TRACE_OBJECT (netsim, "TASK: signal start"); - g_cond_signal (&netsim->start_cond); - g_mutex_unlock (&netsim->loop_mutex); - - GST_TRACE_OBJECT (netsim, "TASK: run"); - g_main_loop_run (loop); - g_main_loop_unref (loop); - - g_mutex_lock (&netsim->loop_mutex); - GST_TRACE_OBJECT (netsim, "TASK: pause"); - gst_pad_pause_task (netsim->srcpad); - netsim->running = FALSE; - GST_TRACE_OBJECT (netsim, "TASK: signal end"); - g_cond_signal (&netsim->start_cond); - g_mutex_unlock (&netsim->loop_mutex); - GST_TRACE_OBJECT (netsim, "TASK: end"); + return nsbuf->arrival_time + nsbuf->delay; } -static gboolean -_main_loop_quit_and_remove_source (gpointer user_data) +static void +net_sim_buffer_free (NetSimBuffer * nsbuf) { - GMainLoop *main_loop = user_data; - GST_DEBUG ("MAINLOOP: Quit %p", main_loop); - g_main_loop_quit (main_loop); - g_assert (!g_main_loop_is_running (main_loop)); - return FALSE; /* Remove source */ + if (G_UNLIKELY (nsbuf == NULL)) + return; + gst_buffer_unref (nsbuf->buf); + g_free (nsbuf); } -static gboolean -gst_net_sim_src_activatemode (GstPad * pad, GstObject * parent, - GstPadMode mode, gboolean active) +static GstFlowReturn +net_sim_buffer_push (NetSimBuffer * nsbuf, GstPad * srcpad) { - GstNetSim *netsim = GST_NET_SIM (parent); - gboolean result = FALSE; - - g_mutex_lock (&netsim->loop_mutex); - if (active) { - if (netsim->main_loop == NULL) { - GMainContext *main_context = g_main_context_new (); - netsim->main_loop = g_main_loop_new (main_context, FALSE); - g_main_context_unref (main_context); - - GST_TRACE_OBJECT (netsim, "ACT: Starting task on srcpad"); - result = gst_pad_start_task (netsim->srcpad, - (GstTaskFunction) gst_net_sim_loop, netsim, NULL); - - GST_TRACE_OBJECT (netsim, "ACT: Wait for task to start"); - g_assert (!netsim->running); - while (!netsim->running) - g_cond_wait (&netsim->start_cond, &netsim->loop_mutex); - GST_TRACE_OBJECT (netsim, "ACT: Task on srcpad started"); - } - } else { - if (netsim->main_loop != NULL) { - GSource *source; - guint id; - - /* Adds an Idle Source which quits the main loop from within. - * This removes the possibility for run/quit race conditions. */ - GST_TRACE_OBJECT (netsim, "DEACT: Stopping main loop on deactivate"); - source = g_idle_source_new (); - g_source_set_callback (source, _main_loop_quit_and_remove_source, - g_main_loop_ref (netsim->main_loop), - (GDestroyNotify) g_main_loop_unref); - id = g_source_attach (source, - g_main_loop_get_context (netsim->main_loop)); - g_source_unref (source); - g_assert_cmpuint (id, >, 0); - g_main_loop_unref (netsim->main_loop); - netsim->main_loop = NULL; - - GST_TRACE_OBJECT (netsim, "DEACT: Wait for mainloop and task to pause"); - g_assert (netsim->running); - while (netsim->running) - g_cond_wait (&netsim->start_cond, &netsim->loop_mutex); - - GST_TRACE_OBJECT (netsim, "DEACT: Stopping task on srcpad"); - result = gst_pad_stop_task (netsim->srcpad); - GST_TRACE_OBJECT (netsim, "DEACT: Mainloop and GstTask stopped"); - } - } - g_mutex_unlock (&netsim->loop_mutex); - - return result; + GstFlowReturn ret; + ret = gst_pad_push (srcpad, nsbuf->buf); + g_free (nsbuf); + return ret; } -typedef struct +static gint +nsbuf_compare_seqnum (gconstpointer a, gconstpointer b, + G_GNUC_UNUSED gpointer user_data) { - GstPad *pad; - GstBuffer *buf; -} PushBufferCtx; + const NetSimBuffer *buf_a = (NetSimBuffer *) a; + const NetSimBuffer *buf_b = (NetSimBuffer *) b; + return buf_a->seqnum - buf_b->seqnum; +} -static inline PushBufferCtx * -push_buffer_ctx_new (GstPad * pad, GstBuffer * buf) +static gint +nsbuf_compare_time (gconstpointer a, gconstpointer b, + G_GNUC_UNUSED gpointer user_data) { - PushBufferCtx *ctx = g_new (PushBufferCtx, 1); - ctx->pad = gst_object_ref (pad); - ctx->buf = gst_buffer_ref (buf); - return ctx; + const NetSimBuffer *buf_a = (NetSimBuffer *) a; + const NetSimBuffer *buf_b = (NetSimBuffer *) b; + GstClockTime ts_a = buf_a->arrival_time + buf_a->delay; + GstClockTime ts_b = buf_b->arrival_time + buf_b->delay; + return ts_a - ts_b; } -static inline void -push_buffer_ctx_free (PushBufferCtx * ctx) +static gboolean +gst_net_sim_set_clock (GstElement * element, GstClock * clock) { - if (G_LIKELY (ctx != NULL)) { - gst_buffer_unref (ctx->buf); - gst_object_unref (ctx->pad); - g_free (ctx); - } + GstNetSim *netsim = GST_NET_SIM_CAST (element); + GST_DEBUG_OBJECT (netsim, "Setting clock %" GST_PTR_FORMAT, clock); + if (clock == NULL) + return TRUE; + + GST_NET_SIM_LOCK (netsim); + if (netsim->clock) + gst_object_unref (netsim->clock); + netsim->clock = gst_object_ref (clock); + GST_NET_SIM_SIGNAL (netsim); + GST_NET_SIM_UNLOCK (netsim); + + return GST_ELEMENT_CLASS (gst_net_sim_parent_class)->set_clock (element, + clock); } static gboolean -push_buffer_ctx_push (PushBufferCtx * ctx) +gst_net_sim_wait_for_clock (GstNetSim * netsim) { - GST_DEBUG_OBJECT (ctx->pad, "Pushing buffer now"); - gst_pad_push (ctx->pad, gst_buffer_ref (ctx->buf)); - return FALSE; + while (netsim->clock == NULL) { + if (!netsim->running) + return FALSE; + GST_INFO_OBJECT (netsim, "Waiting for a clock"); + GST_NET_SIM_WAIT (netsim); + } + return TRUE; } static gint @@ -328,186 +297,399 @@ get_random_value_gamma (GRand * rand_seed, gint32 low, gint32 high, return round (x + low); } -static GstFlowReturn -gst_net_sim_delay_buffer (GstNetSim * netsim, GstBuffer * buf) -{ - GstFlowReturn ret = GST_FLOW_OK; - - g_mutex_lock (&netsim->loop_mutex); - if (netsim->main_loop != NULL && netsim->delay_probability > 0 && - g_rand_double (netsim->rand_seed) < netsim->delay_probability) { - gint delay; - PushBufferCtx *ctx; - GSource *source; - gint64 ready_time, now_time; - - switch (netsim->delay_distribution) { - case DISTRIBUTION_UNIFORM: - delay = get_random_value_uniform (netsim->rand_seed, netsim->min_delay, - netsim->max_delay); - break; - case DISTRIBUTION_NORMAL: - delay = get_random_value_normal (netsim->rand_seed, netsim->min_delay, - netsim->max_delay, &netsim->delay_state); - break; - case DISTRIBUTION_GAMMA: - delay = get_random_value_gamma (netsim->rand_seed, netsim->min_delay, - netsim->max_delay, &netsim->delay_state); - break; - default: - g_assert_not_reached (); - break; - } - - if (delay < 0) - delay = 0; - - ctx = push_buffer_ctx_new (netsim->srcpad, buf); - - source = g_source_new (&gst_net_sim_source_funcs, sizeof (GSource)); - now_time = g_get_monotonic_time (); - ready_time = now_time + delay * 1000; - if (!netsim->allow_reordering && ready_time < netsim->last_ready_time) - ready_time = netsim->last_ready_time + 1; - - netsim->last_ready_time = ready_time; - GST_DEBUG_OBJECT (netsim, "Delaying packet by %" G_GINT64_FORMAT "ms", - (ready_time - now_time) / 1000); - - g_source_set_ready_time (source, ready_time); - g_source_set_callback (source, (GSourceFunc) push_buffer_ctx_push, - ctx, (GDestroyNotify) push_buffer_ctx_free); - g_source_attach (source, g_main_loop_get_context (netsim->main_loop)); - g_source_unref (source); - } else { - ret = gst_pad_push (netsim->srcpad, gst_buffer_ref (buf)); - } - g_mutex_unlock (&netsim->loop_mutex); - - return ret; -} - -static gint -gst_net_sim_get_tokens (GstNetSim * netsim) +static void +gst_net_sim_add_tokens (GstNetSim * netsim, GstClockTime now) { gint tokens = 0; GstClockTimeDiff elapsed_time = 0; - GstClockTime current_time = 0; - GstClockTimeDiff token_time; - GstClock *clock; + + if (netsim->max_kbps == -1 && netsim->max_bucket_size == -1) + return; /* check for umlimited kbps and fill up the bucket if that is the case, * if not, calculate the number of tokens to add based on the elapsed time */ - if (netsim->max_kbps == -1) - return netsim->max_bucket_size * 1000 - netsim->bucket_size; - - /* get the current time */ - clock = gst_element_get_clock (GST_ELEMENT_CAST (netsim)); - if (clock == NULL) { - GST_WARNING_OBJECT (netsim, "No clock, can't get the time"); - } else { - current_time = gst_clock_get_time (clock); + if (netsim->max_kbps == -1) { + netsim->bucket_size = netsim->max_bucket_size * 8; + return; } /* get the elapsed time */ if (GST_CLOCK_TIME_IS_VALID (netsim->prev_time)) { - if (current_time < netsim->prev_time) { + if (now < netsim->prev_time) { GST_WARNING_OBJECT (netsim, "Clock is going backwards!!"); } else { - elapsed_time = GST_CLOCK_DIFF (netsim->prev_time, current_time); + elapsed_time = GST_CLOCK_DIFF (netsim->prev_time, now); } } else { - netsim->prev_time = current_time; + netsim->prev_time = now; } /* calculate number of tokens and how much time is "spent" by these tokens */ tokens = gst_util_uint64_scale_int (elapsed_time, netsim->max_kbps * 1000, GST_SECOND); - token_time = - gst_util_uint64_scale_int (GST_SECOND, tokens, netsim->max_kbps * 1000); - /* increment the time with how much we spent in terms of whole tokens */ - netsim->prev_time += token_time; - gst_object_unref (clock); - return tokens; + GST_DEBUG_OBJECT (netsim, + "Elapsed time: %" GST_TIME_FORMAT " produces %u tokens", + GST_TIME_ARGS (elapsed_time), tokens); + + netsim->prev_time = now; + + netsim->bucket_size = + MIN (netsim->max_bucket_size * 8, netsim->bucket_size + tokens); + GST_LOG_OBJECT (netsim, "Added %d tokens to bucket (contains %u tokens)", + tokens, netsim->bucket_size); } -static gboolean -gst_net_sim_token_bucket (GstNetSim * netsim, GstBuffer * buf) +static GstClockTime +gst_net_sim_get_missing_token_time (GstNetSim * netsim, NetSimBuffer * nsbuf) { - gsize buffer_size; - gint tokens; + GstClockTimeDiff ret; + gint missing_tokens; + gint tokens = nsbuf->size_bits; - /* with an unlimited bucket-size, we have nothing to do */ - if (netsim->max_bucket_size == -1) - return TRUE; + /* unlimited bits per second and unlimited bucket size + means tokens are always available */ + if (netsim->max_kbps == -1 || netsim->max_bucket_size == -1) + return 0; - /* get buffer size in bits */ - buffer_size = gst_buffer_get_size (buf) * 8; - tokens = gst_net_sim_get_tokens (netsim); + /* we we have room in our bucket, no need to wait */ + if (netsim->bucket_size >= tokens) + return 0; - netsim->bucket_size = MIN (G_MAXINT, netsim->bucket_size + tokens); - GST_LOG_OBJECT (netsim, - "Adding %d tokens to bucket (contains %" G_GSIZE_FORMAT " tokens)", - tokens, netsim->bucket_size); + /* lets not divide by 0 */ + if (netsim->max_kbps == 0) + return 0; - if (netsim->max_bucket_size != -1 && netsim->bucket_size > - netsim->max_bucket_size * 1000) - netsim->bucket_size = netsim->max_bucket_size * 1000; + missing_tokens = tokens - netsim->bucket_size; + ret = + gst_util_uint64_scale (GST_SECOND, missing_tokens, + netsim->max_kbps * 1000); - if (buffer_size > netsim->bucket_size) { - GST_DEBUG_OBJECT (netsim, - "Buffer size (%" G_GSIZE_FORMAT ") exeedes bucket size (%" - G_GSIZE_FORMAT ")", buffer_size, netsim->bucket_size); - return FALSE; - } + return ret; +} - netsim->bucket_size -= buffer_size; - GST_LOG_OBJECT (netsim, - "Buffer taking %" G_GSIZE_FORMAT " tokens (%" G_GSIZE_FORMAT " left)", - buffer_size, netsim->bucket_size); - return TRUE; +static void +gst_net_sim_take_tokens (GstNetSim * netsim, NetSimBuffer * nsbuf) +{ + if (netsim->max_kbps == -1 || netsim->max_bucket_size == -1) + return; + + netsim->bucket_size -= nsbuf->size_bits; + GST_DEBUG_OBJECT (netsim, "Buffer taking %u tokens (%d left)", + nsbuf->size_bits, netsim->bucket_size); +} + +static void +gst_net_sim_drop_nsbuf (GstNetSim * netsim) +{ + NetSimBuffer *nsbuf; + nsbuf = g_queue_pop_head (netsim->bqueue); + GST_LOG_OBJECT (netsim, "Dropping buf #%u (tokens)", nsbuf->seqnum); + netsim->bits_in_queue -= nsbuf->size_bits; + net_sim_buffer_free (nsbuf); +} + +static void +gst_net_sim_wait (GstNetSim * netsim, GstClockTime push_time) +{ + netsim->clock_id = gst_clock_new_single_shot_id (netsim->clock, push_time); + + GST_DEBUG_OBJECT (netsim, "waiting for push_time: %" GST_TIME_FORMAT, + GST_TIME_ARGS (push_time)); + + GST_NET_SIM_UNLOCK (netsim); + gst_clock_id_wait (netsim->clock_id, NULL); + GST_NET_SIM_LOCK (netsim); + + gst_clock_id_unref (netsim->clock_id); + netsim->clock_id = NULL; } +/* must be called with GST_NET_SIM_LOCK */ static GstFlowReturn -gst_net_sim_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) +gst_net_sim_push_unlocked (GstNetSim * netsim) { - GstNetSim *netsim = GST_NET_SIM (parent); GstFlowReturn ret = GST_FLOW_OK; + GstClockTime now = gst_clock_get_time (netsim->clock); + NetSimBuffer *nsbuf = g_queue_peek_head (netsim->bqueue); + GstClockTime push_time = net_sim_buffer_get_push_time (nsbuf); + GstClockTime token_delay; + + gst_net_sim_add_tokens (netsim, now); + + GST_DEBUG_OBJECT (netsim, + "now: %" GST_TIME_FORMAT " netsim->max_delay %" GST_STIME_FORMAT, + GST_TIME_ARGS (now), GST_STIME_ARGS (netsim->max_delay * GST_MSECOND)); + + token_delay = gst_net_sim_get_missing_token_time (netsim, nsbuf); + if (token_delay > 0) { + GstClockTime deadline = now + token_delay; + GstClockTimeDiff delay = GST_CLOCK_DIFF (push_time, deadline); + + /* if delay is too big, we drop */ + if (delay > netsim->max_queue_delay * GST_MSECOND) { + GST_DEBUG_OBJECT (netsim, + "Delay %ums > max_queue_delay %ums, dropping buffer", + (guint) GST_TIME_AS_MSECONDS (delay), netsim->max_queue_delay); + gst_net_sim_drop_nsbuf (netsim); + return GST_FLOW_OK; + } else { + /* if not, we wait until the deadline and then push */ + GST_DEBUG_OBJECT (netsim, + "delaying buffer #%u an additional %" GST_TIME_FORMAT + "for new deadline: %" GST_TIME_FORMAT, + nsbuf->seqnum, GST_TIME_ARGS (token_delay), GST_TIME_ARGS (deadline)); + push_time += token_delay; + } + } + + gst_net_sim_wait (netsim, push_time); - if (!gst_net_sim_token_bucket (netsim, buf)) - goto done; + GST_DEBUG_OBJECT (netsim, "Pushing buffer #%u now", nsbuf->seqnum); + nsbuf = g_queue_pop_head (netsim->bqueue); + gst_net_sim_take_tokens (netsim, nsbuf); + netsim->bits_in_queue -= nsbuf->size_bits; + + GST_NET_SIM_UNLOCK (netsim); + ret = net_sim_buffer_push (nsbuf, netsim->srcpad); + GST_NET_SIM_LOCK (netsim); + + return ret; +} + +static gint +gst_new_sim_get_delay_ms (GstNetSim * netsim) +{ + gint delay_ms = 0; + + if (netsim->delay_probability == 0 || + g_rand_double (netsim->rand_seed) > netsim->delay_probability) + return delay_ms; + + switch (netsim->delay_distribution) { + case DISTRIBUTION_UNIFORM: + delay_ms = get_random_value_uniform (netsim->rand_seed, netsim->min_delay, + netsim->max_delay); + break; + case DISTRIBUTION_NORMAL: + delay_ms = get_random_value_normal (netsim->rand_seed, netsim->min_delay, + netsim->max_delay, &netsim->delay_state); + break; + case DISTRIBUTION_GAMMA: + delay_ms = get_random_value_gamma (netsim->rand_seed, netsim->min_delay, + netsim->max_delay, &netsim->delay_state); + break; + default: + g_assert_not_reached (); + break; + } + + return delay_ms; +} + +static gint +gst_new_sim_get_throttle_ms (GstNetSim * netsim, GstClockTime now) +{ + gint ret = 0; + GstClockTime start_time; + + /* we don't have a frequency or a delay, return 0 */ + if (!netsim->throttle_frequency || !netsim->throttle_delay) { + return 0; + } + + /* Invalid configuration */ + if (netsim->throttle_delay / 1000 > netsim->throttle_frequency) { + GST_ERROR_OBJECT (netsim, + "throttle-delay larger than frequency not allowed!"); + g_assert_not_reached (); + } + + /* we are not valid yet, set it to now */ + if (!GST_CLOCK_TIME_IS_VALID (netsim->throttle_end_time)) { + netsim->throttle_end_time = now; + } + + /* we are past our end-time, move it forward */ + if (now >= netsim->throttle_end_time) { + netsim->throttle_end_time += (netsim->throttle_frequency * GST_SECOND); + } + + /* when the throttling should start */ + start_time = + netsim->throttle_end_time - (netsim->throttle_delay * GST_MSECOND); + + /* we are within the throttling window, delay all packets until throttle_end_time */ + if (now >= start_time) { + ret = GST_TIME_AS_MSECONDS (netsim->throttle_end_time - now); + } + + return ret; +} + +static void +gst_net_sim_queue_buffer (GstNetSim * netsim, GstBuffer * buf) +{ + GstClockTime now; + + if (!netsim->clock) { + GST_WARNING_OBJECT (netsim, "No clock, dropping buffer"); + return; + } + + now = gst_clock_get_time (netsim->clock); + gint delay_ms = gst_new_sim_get_delay_ms (netsim); + delay_ms += gst_new_sim_get_throttle_ms (netsim, now); + + NetSimBuffer *nsbuf = net_sim_buffer_new (buf, netsim->seqnum++, + now, delay_ms * GST_MSECOND); + + if (delay_ms > 0) { + GST_DEBUG_OBJECT (netsim, "Delaying buffer with %dms", delay_ms); + } + + GST_DEBUG_OBJECT (netsim, "queue_size: %d, bits_in_queue: %u, bufsize: %u", + netsim->queue_size * 1000, netsim->bits_in_queue, nsbuf->size_bits); + + if (netsim->bits_in_queue > 0 && + netsim->queue_size != -1 && + netsim->bits_in_queue + nsbuf->size_bits > netsim->queue_size * 1000) { + GST_DEBUG_OBJECT (netsim, "dropping buf #%u", nsbuf->seqnum); + net_sim_buffer_free (nsbuf); + } else { + GST_DEBUG_OBJECT (netsim, "queueing buf #%u", nsbuf->seqnum); + GST_NET_SIM_LOCK (netsim); + g_queue_insert_sorted (netsim->bqueue, nsbuf, netsim->compare_func, NULL); + netsim->bits_in_queue += nsbuf->size_bits; + GST_NET_SIM_SIGNAL (netsim); + GST_NET_SIM_UNLOCK (netsim); + } +} + +static GstFlowReturn +gst_net_sim_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) +{ + GstNetSim *netsim = GST_NET_SIM_CAST (parent); + gboolean dropped = FALSE; if (netsim->drop_packets > 0) { netsim->drop_packets--; GST_DEBUG_OBJECT (netsim, "Dropping packet (%d left)", netsim->drop_packets); + dropped = TRUE; } else if (netsim->drop_probability > 0 && g_rand_double (netsim->rand_seed) < (gdouble) netsim->drop_probability) { GST_DEBUG_OBJECT (netsim, "Dropping packet"); + dropped = TRUE; } else if (netsim->duplicate_probability > 0 && g_rand_double (netsim->rand_seed) < (gdouble) netsim->duplicate_probability) { GST_DEBUG_OBJECT (netsim, "Duplicating packet"); - gst_net_sim_delay_buffer (netsim, buf); - ret = gst_net_sim_delay_buffer (netsim, buf); + gst_net_sim_queue_buffer (netsim, buf); + gst_net_sim_queue_buffer (netsim, buf); } else { - ret = gst_net_sim_delay_buffer (netsim, buf); + gst_net_sim_queue_buffer (netsim, buf); + } + + if (dropped && netsim->replace_droppped_with_empty) { + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + + if (gst_rtp_buffer_map (buf, GST_MAP_READ, &rtp)) { + guint header_len = gst_rtp_buffer_get_header_len (&rtp); + gst_rtp_buffer_unmap (&rtp); + + buf = gst_buffer_make_writable (buf); + gst_buffer_resize (buf, 0, header_len); + gst_net_sim_queue_buffer (netsim, buf); + } } -done: gst_buffer_unref (buf); - return ret; + + return GST_FLOW_OK; } +static void +gst_net_sim_loop (gpointer data) +{ + GstNetSim *netsim = GST_NET_SIM_CAST (data); + GstFlowReturn ret; + + GST_NET_SIM_LOCK (netsim); + if (!gst_net_sim_wait_for_clock (netsim)) + goto pause_task; + + if (!netsim->running) + goto pause_task; + + while (netsim->running && g_queue_is_empty (netsim->bqueue)) + GST_NET_SIM_WAIT (netsim); + + if (!netsim->running) + goto pause_task; + + ret = gst_net_sim_push_unlocked (netsim); + if (ret != GST_FLOW_OK) { + GST_ERROR_OBJECT (netsim, "pausing task because flow: %d", ret); + goto pause_task; + } + + if (!netsim->running) { + goto pause_task; + } + + GST_NET_SIM_UNLOCK (netsim); + return; + +pause_task: + GST_INFO_OBJECT (netsim, "pausing task"); + gst_pad_pause_task (netsim->srcpad); + GST_NET_SIM_UNLOCK (netsim); + return; +} + +static gboolean +gst_net_sim_src_activatemode (GstPad * pad, GstObject * parent, + GstPadMode mode, gboolean active) +{ + GstNetSim *netsim = GST_NET_SIM_CAST (parent); + gboolean result; + + if (active) { + GST_DEBUG_OBJECT (pad, "starting task on pad %s:%s", + GST_DEBUG_PAD_NAME (pad)); + + GST_NET_SIM_LOCK (netsim); + netsim->running = TRUE; + if (!gst_pad_start_task (netsim->srcpad, gst_net_sim_loop, netsim, NULL)) + g_assert_not_reached (); + + GST_NET_SIM_UNLOCK (netsim); + result = TRUE; + } else { + GST_DEBUG_OBJECT (pad, "stopping task on pad %s:%s", + GST_DEBUG_PAD_NAME (pad)); + + GST_NET_SIM_LOCK (netsim); + netsim->running = FALSE; + if (netsim->clock_id) + gst_clock_id_unschedule (netsim->clock_id); + GST_NET_SIM_SIGNAL (netsim); + GST_NET_SIM_UNLOCK (netsim); + + result = gst_pad_stop_task (pad); + } + + return result; +} static void gst_net_sim_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { - GstNetSim *netsim = GST_NET_SIM (object); + GstNetSim *netsim = GST_NET_SIM_CAST (object); switch (prop_id) { case PROP_MIN_DELAY: @@ -537,10 +719,29 @@ gst_net_sim_set_property (GObject * object, case PROP_MAX_BUCKET_SIZE: netsim->max_bucket_size = g_value_get_int (value); if (netsim->max_bucket_size != -1) - netsim->bucket_size = netsim->max_bucket_size * 1000; + netsim->bucket_size = netsim->max_bucket_size * 8; + break; + case PROP_QUEUE_SIZE: + netsim->queue_size = g_value_get_int (value); + break; + case PROP_MAX_QUEUE_DELAY: + netsim->max_queue_delay = g_value_get_int (value); break; case PROP_ALLOW_REORDERING: netsim->allow_reordering = g_value_get_boolean (value); + if (netsim->allow_reordering) + netsim->compare_func = nsbuf_compare_time; + else + netsim->compare_func = nsbuf_compare_seqnum; + break; + case PROP_REPLACE_DROPPED_WITH_EMPTY: + netsim->replace_droppped_with_empty = g_value_get_boolean (value); + break; + case PROP_THROTTLE_FREQUENCY: + netsim->throttle_frequency = g_value_get_int (value); + break; + case PROP_THROTTLE_DELAY: + netsim->throttle_delay = g_value_get_int (value); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -552,7 +753,7 @@ static void gst_net_sim_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { - GstNetSim *netsim = GST_NET_SIM (object); + GstNetSim *netsim = GST_NET_SIM_CAST (object); switch (prop_id) { case PROP_MIN_DELAY: @@ -582,9 +783,24 @@ gst_net_sim_get_property (GObject * object, case PROP_MAX_BUCKET_SIZE: g_value_set_int (value, netsim->max_bucket_size); break; + case PROP_QUEUE_SIZE: + g_value_set_int (value, netsim->queue_size); + break; + case PROP_MAX_QUEUE_DELAY: + g_value_set_int (value, netsim->max_queue_delay); + break; case PROP_ALLOW_REORDERING: g_value_set_boolean (value, netsim->allow_reordering); break; + case PROP_REPLACE_DROPPED_WITH_EMPTY: + g_value_set_boolean (value, netsim->replace_droppped_with_empty); + break; + case PROP_THROTTLE_FREQUENCY: + g_value_set_int (value, netsim->throttle_frequency); + break; + case PROP_THROTTLE_DELAY: + g_value_set_int (value, netsim->throttle_delay); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -603,11 +819,12 @@ gst_net_sim_init (GstNetSim * netsim) gst_element_add_pad (GST_ELEMENT (netsim), netsim->srcpad); gst_element_add_pad (GST_ELEMENT (netsim), netsim->sinkpad); - g_mutex_init (&netsim->loop_mutex); - g_cond_init (&netsim->start_cond); + netsim->bqueue = g_queue_new (); + g_mutex_init (&netsim->mutex); + g_cond_init (&netsim->cond); netsim->rand_seed = g_rand_new (); - netsim->main_loop = NULL; netsim->prev_time = GST_CLOCK_TIME_NONE; + netsim->throttle_end_time = GST_CLOCK_TIME_NONE; GST_OBJECT_FLAG_SET (netsim->sinkpad, GST_PAD_FLAG_PROXY_CAPS | GST_PAD_FLAG_PROXY_ALLOCATION); @@ -621,23 +838,17 @@ gst_net_sim_init (GstNetSim * netsim) static void gst_net_sim_finalize (GObject * object) { - GstNetSim *netsim = GST_NET_SIM (object); + GstNetSim *netsim = GST_NET_SIM_CAST (object); g_rand_free (netsim->rand_seed); - g_mutex_clear (&netsim->loop_mutex); - g_cond_clear (&netsim->start_cond); + g_mutex_clear (&netsim->mutex); + g_cond_clear (&netsim->cond); - G_OBJECT_CLASS (gst_net_sim_parent_class)->finalize (object); -} + gst_object_replace ((GstObject **) & netsim->clock, NULL); -static void -gst_net_sim_dispose (GObject * object) -{ - GstNetSim *netsim = GST_NET_SIM (object); + g_queue_free_full (netsim->bqueue, (GDestroyNotify) net_sim_buffer_free); - g_assert (netsim->main_loop == NULL); - - G_OBJECT_CLASS (gst_net_sim_parent_class)->dispose (object); + G_OBJECT_CLASS (gst_net_sim_parent_class)->finalize (object); } static void @@ -659,11 +870,11 @@ gst_net_sim_class_init (GstNetSimClass * klass) "Philippe Kalaf , " "Havard Graff "); - gobject_class->dispose = GST_DEBUG_FUNCPTR (gst_net_sim_dispose); gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_net_sim_finalize); gobject_class->set_property = gst_net_sim_set_property; gobject_class->get_property = gst_net_sim_get_property; + gstelement_class->set_clock = GST_DEBUG_FUNCPTR (gst_net_sim_set_clock); g_object_class_install_property (gobject_class, PROP_MIN_DELAY, g_param_spec_int ("min-delay", "Minimum delay (ms)", @@ -713,6 +924,15 @@ gst_net_sim_class_init (GstNetSimClass * klass) "Drop the next n packets", 0, G_MAXUINT, DEFAULT_DROP_PACKETS, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_REPLACE_DROPPED_WITH_EMPTY, + g_param_spec_boolean ("replace-dropped-with-empty-packets", + "replace-dropped-with-empty-packets", + "Insert packets with no payload instead of dropping", + DEFAULT_REPLACE_DROPPED_WITH_EMPTY, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + /** * GstNetSim:max-kbps: * @@ -731,16 +951,44 @@ gst_net_sim_class_init (GstNetSimClass * klass) /** * GstNetSim:max-bucket-size: * - * The size of the token bucket, related to burstiness resilience. + * The size of the token bucket in Bytes. * * Since: 1.14 */ g_object_class_install_property (gobject_class, PROP_MAX_BUCKET_SIZE, - g_param_spec_int ("max-bucket-size", "Maximum Bucket Size (Kb)", - "The size of the token bucket, related to burstiness resilience " + g_param_spec_int ("max-bucket-size", "Maximum Bucket Size (B)", + "The size of the token bucket, in bytes " "(-1 = unlimited)", -1, G_MAXINT, DEFAULT_MAX_BUCKET_SIZE, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + /** + * GstNetSim:queue-size: + * + * In case of insufficient tokens in the bucket, this property says how + * much bits we can keep in the queue + * + * Since: 1.18 + */ + g_object_class_install_property (gobject_class, PROP_QUEUE_SIZE, + g_param_spec_int ("queue-size", "Queue Size in bits", + "The max number of bits in the internal queue " + "(-1 = unlimited)", + -1, G_MAXINT, DEFAULT_QUEUE_SIZE, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + + /** + * GstNetSim:max-queue-delay: + * + * How long we will delay packets for in the queue + * + * Since: 1.18 + */ + g_object_class_install_property (gobject_class, PROP_MAX_QUEUE_DELAY, + g_param_spec_int ("max-queue-delay", "Maximum queue delay (ms)", + "The maximum delay a buffer can be given before being dropped", + G_MININT, G_MAXINT, DEFAULT_MAX_QUEUE_DELAY, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + /** * GstNetSim:allow-reordering: * @@ -757,6 +1005,19 @@ gst_net_sim_class_init (GstNetSimClass * klass) DEFAULT_ALLOW_REORDERING, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_THROTTLE_FREQUENCY, + g_param_spec_int ("throttle-frequency", "Throttle Frequency (s)", + "How often (in seconds) a throttle should occur (0 = never)", + 0, G_MAXINT, DEFAULT_THROTTLE_FREQUENCY, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_THROTTLE_DELAY, + g_param_spec_int ("throttle-delay", "Throttle Delay (ms)", + "When a throttle occurs, for how long the packets should be held back", + 0, G_MAXINT, DEFAULT_THROTTLE_DELAY, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + GST_DEBUG_CATEGORY_INIT (netsim_debug, "netsim", 0, "Network simulator"); gst_type_mark_as_plugin_api (distribution_get_type (), 0); diff --git a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h index cbd627012d2..066393e4844 100644 --- a/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h +++ b/subprojects/gst-plugins-bad/gst/netsim/gstnetsim.h @@ -42,6 +42,7 @@ G_BEGIN_DECLS (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_NET_SIM)) #define GST_IS_NET_SIM_CLASS(obj) \ (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_NET_SIM)) +#define GST_NET_SIM_CAST(obj) ((GstNetSim *)obj) typedef struct _GstNetSim GstNetSim; typedef struct _GstNetSimClass GstNetSimClass; @@ -67,15 +68,21 @@ struct _GstNetSim GstPad *sinkpad; GstPad *srcpad; - GMutex loop_mutex; - GCond start_cond; - GMainLoop *main_loop; - gboolean running; GRand *rand_seed; - gsize bucket_size; - GstClockTime prev_time; + gint bucket_size; NormalDistributionState delay_state; - gint64 last_ready_time; + GstClockTime prev_time; + + GstClock *clock; + GstClockID clock_id; + GMutex mutex; + GCond cond; + gboolean running; + GQueue *bqueue; + guint seqnum; + GCompareDataFunc compare_func; + guint bits_in_queue; + GstClockTime throttle_end_time; /* properties */ gint min_delay; @@ -87,7 +94,12 @@ struct _GstNetSim guint drop_packets; gint max_kbps; gint max_bucket_size; + gint queue_size; + gint max_queue_delay; gboolean allow_reordering; + gboolean replace_droppped_with_empty; + gint throttle_frequency; + gint throttle_delay; }; struct _GstNetSimClass diff --git a/subprojects/gst-plugins-bad/gst/netsim/meson.build b/subprojects/gst-plugins-bad/gst/netsim/meson.build index 27a5ca4f9f8..78ac68f3a24 100644 --- a/subprojects/gst-plugins-bad/gst/netsim/meson.build +++ b/subprojects/gst-plugins-bad/gst/netsim/meson.build @@ -6,7 +6,7 @@ gstnetsim = library('gstnetsim', netsim_sources, c_args : gst_plugins_bad_args, include_directories : [configinc], - dependencies : [gstbase_dep, libm], + dependencies : [gstbase_dep, gstrtp_dep, libm], install : true, install_dir : plugins_install_dir, ) diff --git a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c index 53e7ef6fc25..6b409ae0ce1 100644 --- a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c +++ b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.c @@ -46,6 +46,8 @@ #include #endif +#define GLIB_DISABLE_DEPRECATION_WARNINGS + #include "gstpcapparse.h" #include @@ -56,6 +58,8 @@ #include #else #include +#include +#include #endif @@ -73,9 +77,58 @@ enum PROP_SRC_PORT, PROP_DST_PORT, PROP_CAPS, - PROP_TS_OFFSET + PROP_TS_OFFSET, + PROP_STATS, + PROP_START_TIME, +}; + +typedef enum +{ + PCAP_PARSE_STATE_CREATED, + PCAP_PARSE_STATE_PARSING, +} GstPcapParseState; + +typedef enum +{ + LINKTYPE_ETHER = 1, + LINKTYPE_RAW = 101, + LINKTYPE_SLL = 113, + LINKTYPE_SLL2 = 276 +} GstPcapParseLinktype; + +struct _GstPcapParse +{ + GstElement element; + + /*< private > */ + GstPad *sink_pad; + GstPad *src_pad; + + /* properties */ + gchar *src_ip; + gchar *dst_ip; + gint src_port; + gint dst_port; + GstCaps *caps; + gint64 offset; + GstClockTime start_ts; + + /* state */ + GstAdapter * adapter; + GHashTable *stats_map; + gboolean initialized; + gboolean swap_endian; + gboolean nanosecond_timestamp; + gint64 cur_packet_size; + GstClockTime cur_ts; + GstPcapParseLinktype linktype; + + gboolean newsegment_sent; + gboolean first_packet; + gboolean discont; }; + GST_DEBUG_CATEGORY_STATIC (gst_pcap_parse_debug); #define GST_CAT_DEFAULT gst_pcap_parse_debug @@ -89,231 +142,27 @@ static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY); -static void gst_pcap_parse_finalize (GObject * object); -static void gst_pcap_parse_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec); -static void gst_pcap_parse_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec); -static GstStateChangeReturn -gst_pcap_parse_change_state (GstElement * element, GstStateChange transition); - -static void gst_pcap_parse_reset (GstPcapParse * self); - -static GstFlowReturn gst_pcap_parse_chain (GstPad * pad, - GstObject * parent, GstBuffer * buffer); -static gboolean gst_pcap_sink_event (GstPad * pad, - GstObject * parent, GstEvent * event); - - #define parent_class gst_pcap_parse_parent_class G_DEFINE_TYPE (GstPcapParse, gst_pcap_parse, GST_TYPE_ELEMENT); GST_ELEMENT_REGISTER_DEFINE (pcapparse, "pcapparse", GST_RANK_NONE, GST_TYPE_PCAP_PARSE); -static void -gst_pcap_parse_class_init (GstPcapParseClass * klass) -{ - GObjectClass *gobject_class = G_OBJECT_CLASS (klass); - GstElementClass *element_class = GST_ELEMENT_CLASS (klass); - - gobject_class->finalize = gst_pcap_parse_finalize; - gobject_class->get_property = gst_pcap_parse_get_property; - gobject_class->set_property = gst_pcap_parse_set_property; - - g_object_class_install_property (gobject_class, - PROP_SRC_IP, g_param_spec_string ("src-ip", "Source IP", - "Source IP to restrict to", "", - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property (gobject_class, - PROP_DST_IP, g_param_spec_string ("dst-ip", "Destination IP", - "Destination IP to restrict to", "", - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property (gobject_class, - PROP_SRC_PORT, g_param_spec_int ("src-port", "Source port", - "Source port to restrict to", -1, G_MAXUINT16, -1, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property (gobject_class, - PROP_DST_PORT, g_param_spec_int ("dst-port", "Destination port", - "Destination port to restrict to", -1, G_MAXUINT16, -1, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property (gobject_class, PROP_CAPS, - g_param_spec_boxed ("caps", "Caps", - "The caps of the source pad", GST_TYPE_CAPS, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - g_object_class_install_property (gobject_class, PROP_TS_OFFSET, - g_param_spec_int64 ("ts-offset", "Timestamp Offset", - "Relative timestamp offset (ns) to apply (-1 = use absolute packet time)", - -1, G_MAXINT64, -1, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - gst_element_class_add_static_pad_template (element_class, &sink_template); - gst_element_class_add_static_pad_template (element_class, &src_template); - - element_class->change_state = gst_pcap_parse_change_state; - - gst_element_class_set_static_metadata (element_class, "PCapParse", - "Raw/Parser", - "Parses a raw pcap stream", - "Ole André Vadla Ravnås "); - - GST_DEBUG_CATEGORY_INIT (gst_pcap_parse_debug, "pcapparse", 0, "pcap parser"); -} - -static void -gst_pcap_parse_init (GstPcapParse * self) -{ - self->sink_pad = gst_pad_new_from_static_template (&sink_template, "sink"); - gst_pad_set_chain_function (self->sink_pad, - GST_DEBUG_FUNCPTR (gst_pcap_parse_chain)); - gst_pad_use_fixed_caps (self->sink_pad); - gst_pad_set_event_function (self->sink_pad, - GST_DEBUG_FUNCPTR (gst_pcap_sink_event)); - gst_element_add_pad (GST_ELEMENT (self), self->sink_pad); - - self->src_pad = gst_pad_new_from_static_template (&src_template, "src"); - gst_pad_use_fixed_caps (self->src_pad); - gst_element_add_pad (GST_ELEMENT (self), self->src_pad); - - self->src_ip = -1; - self->dst_ip = -1; - self->src_port = -1; - self->dst_port = -1; - self->offset = -1; - - self->adapter = gst_adapter_new (); - - gst_pcap_parse_reset (self); -} - -static void -gst_pcap_parse_finalize (GObject * object) -{ - GstPcapParse *self = GST_PCAP_PARSE (object); - - g_object_unref (self->adapter); - if (self->caps) - gst_caps_unref (self->caps); - - G_OBJECT_CLASS (parent_class)->finalize (object); -} - -static const gchar * -get_ip_address_as_string (gint64 ip_addr) -{ - if (ip_addr >= 0) { - struct in_addr addr; - addr.s_addr = ip_addr; - return inet_ntoa (addr); - } else { - return ""; - } -} - -static void -set_ip_address_from_string (gint64 * ip_addr, const gchar * ip_str) -{ - if (ip_str[0] != '\0') { - gulong addr = inet_addr (ip_str); - if (addr != INADDR_NONE) - *ip_addr = addr; - } else { - *ip_addr = -1; - } -} - -static void -gst_pcap_parse_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec) -{ - GstPcapParse *self = GST_PCAP_PARSE (object); - - switch (prop_id) { - case PROP_SRC_IP: - g_value_set_string (value, get_ip_address_as_string (self->src_ip)); - break; - - case PROP_DST_IP: - g_value_set_string (value, get_ip_address_as_string (self->dst_ip)); - break; - - case PROP_SRC_PORT: - g_value_set_int (value, self->src_port); - break; - - case PROP_DST_PORT: - g_value_set_int (value, self->dst_port); - break; - - case PROP_CAPS: - gst_value_set_caps (value, self->caps); - break; - - case PROP_TS_OFFSET: - g_value_set_int64 (value, self->offset); - break; +#define ETH_HEADER_LEN 14 +#define SLL_HEADER_LEN 16 +#define IP_HEADER_MIN_LEN 20 +#define UDP_HEADER_LEN 8 - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} +#define IP_PROTO_UDP 17 +#define IP_PROTO_TCP 6 -static void -gst_pcap_parse_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec) +static gchar * +get_ip_address_as_string (guint32 ip_addr) { - GstPcapParse *self = GST_PCAP_PARSE (object); - - switch (prop_id) { - case PROP_SRC_IP: - set_ip_address_from_string (&self->src_ip, g_value_get_string (value)); - break; - - case PROP_DST_IP: - set_ip_address_from_string (&self->dst_ip, g_value_get_string (value)); - break; - - case PROP_SRC_PORT: - self->src_port = g_value_get_int (value); - break; - - case PROP_DST_PORT: - self->dst_port = g_value_get_int (value); - break; - - case PROP_CAPS: - { - const GstCaps *new_caps_val; - GstCaps *new_caps, *old_caps; - - new_caps_val = gst_value_get_caps (value); - if (new_caps_val == NULL) { - new_caps = gst_caps_new_any (); - } else { - new_caps = gst_caps_copy (new_caps_val); - } - - old_caps = self->caps; - self->caps = new_caps; - if (old_caps) - gst_caps_unref (old_caps); - - gst_pad_set_caps (self->src_pad, new_caps); - break; - } - - case PROP_TS_OFFSET: - self->offset = g_value_get_int64 (value); - break; - - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } + char ip_str[INET_ADDRSTRLEN]; + struct in_addr addr; + addr.s_addr = ip_addr; + inet_ntop (AF_INET, &addr, ip_str, INET_ADDRSTRLEN); + return g_strdup (ip_str); } static void @@ -324,11 +173,13 @@ gst_pcap_parse_reset (GstPcapParse * self) self->nanosecond_timestamp = FALSE; self->cur_packet_size = -1; self->cur_ts = GST_CLOCK_TIME_NONE; - self->base_ts = GST_CLOCK_TIME_NONE; + self->start_ts = GST_CLOCK_TIME_NONE; self->newsegment_sent = FALSE; self->first_packet = TRUE; + self->discont = TRUE; gst_adapter_clear (self->adapter); + g_hash_table_remove_all (self->stats_map); } static guint32 @@ -355,15 +206,161 @@ gst_pcap_parse_read_uint32 (GstPcapParse * self, const guint8 * p) #define IP_HEADER_MIN_LEN 20 #define UDP_HEADER_LEN 8 -#define IP_PROTO_UDP 17 -#define IP_PROTO_TCP 6 +static GValueArray * +gst_pcap_parse_get_stats (GstPcapParse * self) +{ + GValueArray *ret; + GList *streams, *walk; + guint len, i; + + len = g_hash_table_size (self->stats_map); + ret = g_value_array_new (len); + + walk = streams = g_hash_table_get_values (self->stats_map); + for (i = 0; i < len; i++) { + GstStructure *s = walk->data; + GValue *value; + ret->n_values++; + value = g_value_array_get_nth (ret, i); + GST_INFO_OBJECT (self, "Adding stats %d: %" GST_PTR_FORMAT, i, s); + + g_value_init (value, GST_TYPE_STRUCTURE); + gst_value_set_structure (value, s); + + walk = walk->next; + } + + g_list_free (streams); + + return ret; +} + +/* from gstrtpbuffer.c */ +typedef struct _GstRTPHeader +{ +#if G_BYTE_ORDER == G_LITTLE_ENDIAN + unsigned int csrc_count:4; /* CSRC count */ + unsigned int extension:1; /* header extension flag */ + unsigned int padding:1; /* padding flag */ + unsigned int version:2; /* protocol version */ + unsigned int payload_type:7; /* payload type */ + unsigned int marker:1; /* marker bit */ +#elif G_BYTE_ORDER == G_BIG_ENDIAN + unsigned int version:2; /* protocol version */ + unsigned int padding:1; /* padding flag */ + unsigned int extension:1; /* header extension flag */ + unsigned int csrc_count:4; /* CSRC count */ + unsigned int marker:1; /* marker bit */ + unsigned int payload_type:7; /* payload type */ +#else +#error "G_BYTE_ORDER should be big or little endian." +#endif + unsigned int seq:16; /* sequence number */ + unsigned int timestamp:32; /* timestamp */ + unsigned int ssrc:32; /* synchronization source */ + guint8 csrclist[4]; /* optional CSRC list, 32 bits each */ +} GstRTPHeader; + +static void +_check_rtp_rtcp (const guint8 * payload, gint payload_size, + gboolean * is_rtp, gboolean * is_rtcp, gint * payload_type, guint32 * ssrc) +{ + GstRTPHeader *rtp; + *is_rtp = FALSE; + *is_rtcp = FALSE; + + /* minimum rtp-header length */ + if (payload_size < 12) + return; + + rtp = (GstRTPHeader *) payload; + /* check version (common for both RTP & RTCP) */ + if (rtp->version != 2) + return; + + /* for payload_type range 66-95 we assume RTCP, not RTP */ + if (rtp->payload_type >= 66 && rtp->payload_type <= 95) { + *is_rtcp = TRUE; + *ssrc = rtp->timestamp; /* Hackish, but true */ + } else { + *payload_type = rtp->payload_type; + *ssrc = rtp->ssrc; + *is_rtp = TRUE; + } +} + +static void +_add_rtp_stats (GstStructure * s, const guint8 * payload, gint payload_size) +{ + gboolean is_rtp; + gboolean is_rtcp; + gint payload_type; + guint32 ssrc; + + _check_rtp_rtcp (payload, payload_size, &is_rtp, &is_rtcp, &payload_type, + &ssrc); + + if (is_rtcp) { + gst_structure_set (s, "has-rtcp", G_TYPE_BOOLEAN, TRUE, NULL); + + gst_structure_set (s, "ssrc", G_TYPE_UINT, ssrc, NULL); + + } else if (is_rtp) { + gst_structure_set (s, "has-rtp", G_TYPE_BOOLEAN, TRUE, NULL); + + /* FIXME: support multiple pt/ssrc */ + gst_structure_set (s, + "payload-type", G_TYPE_INT, payload_type, + "ssrc", G_TYPE_UINT, ssrc, NULL); + } +} + +static void +gst_pcap_parse_add_stats (GstPcapParse * self, + const guint8 * payload, gint payload_size, + const gchar * src_ip, guint16 src_port, + const gchar * dst_ip, guint16 dst_port) +{ + GstStructure *s; + gint packets; + gint bytes; + + gchar *key_str = g_strdup_printf ("%s:%d->%s:%d", + src_ip, src_port, dst_ip, dst_port); + + s = g_hash_table_lookup (self->stats_map, key_str); + if (s == NULL) { + s = gst_structure_new ("stats", + "first-ts", G_TYPE_UINT64, self->cur_ts, + "id-str", G_TYPE_STRING, key_str, + "src-ip", G_TYPE_STRING, src_ip, + "src-port", G_TYPE_INT, src_port, + "dst-ip", G_TYPE_STRING, dst_ip, + "dst-port", G_TYPE_INT, dst_port, + "packets", G_TYPE_INT, 0, "bytes", G_TYPE_INT, 0, NULL); + g_hash_table_insert (self->stats_map, g_strdup (key_str), s); + } + g_free (key_str); + + gst_structure_get (s, + "packets", G_TYPE_INT, &packets, "bytes", G_TYPE_INT, &bytes, NULL); + + packets += 1; + bytes += payload_size; + + gst_structure_set (s, + "packets", G_TYPE_INT, packets, "bytes", G_TYPE_INT, bytes, NULL); + + _add_rtp_stats (s, payload, payload_size); +} static gboolean gst_pcap_parse_scan_frame (GstPcapParse * self, const guint8 * buf, gint buf_size, const guint8 ** payload, gint * payload_size) { + gboolean ret = FALSE; const guint8 *buf_ip = 0; const guint8 *buf_proto; guint16 eth_type; @@ -374,6 +371,8 @@ gst_pcap_parse_scan_frame (GstPcapParse * self, guint8 ip_protocol; guint32 ip_src_addr; guint32 ip_dst_addr; + gchar *src_ip = NULL; + gchar *dst_ip = NULL; guint16 src_port; guint16 dst_port; guint16 len; @@ -382,14 +381,14 @@ gst_pcap_parse_scan_frame (GstPcapParse * self, switch (self->linktype) { case LINKTYPE_ETHER: if (buf_size < ETH_HEADER_LEN + IP_HEADER_MIN_LEN + UDP_HEADER_LEN) - return FALSE; + goto done; eth_type = GUINT16_FROM_BE (*((guint16 *) (buf + ETH_MAC_ADDRESSES_LEN))); /* check for vlan 802.1q header (4 bytes, with first two bytes equal to 0x8100) */ if (eth_type == 0x8100) { if (buf_size < ETH_HEADER_LEN + ETH_VLAN_HEADER_LEN + IP_HEADER_MIN_LEN + UDP_HEADER_LEN) - return FALSE; + goto done; eth_type = GUINT16_FROM_BE (*((guint16 *) (buf + ETH_MAC_ADDRESSES_LEN + ETH_VLAN_HEADER_LEN))); @@ -400,7 +399,7 @@ gst_pcap_parse_scan_frame (GstPcapParse * self, break; case LINKTYPE_SLL: if (buf_size < SLL_HEADER_LEN + IP_HEADER_MIN_LEN + UDP_HEADER_LEN) - return FALSE; + goto done; eth_type = GUINT16_FROM_BE (*((guint16 *) (buf + 14))); buf_ip = buf + SLL_HEADER_LEN; @@ -414,32 +413,32 @@ gst_pcap_parse_scan_frame (GstPcapParse * self, break; case LINKTYPE_RAW: if (buf_size < IP_HEADER_MIN_LEN + UDP_HEADER_LEN) - return FALSE; + goto done; eth_type = 0x800; /* This is fine since IPv4/IPv6 is parse elsewhere */ buf_ip = buf; break; default: - return FALSE; + goto done; } if (eth_type != 0x800) { GST_ERROR_OBJECT (self, "Link type %d: Ethernet type %d is not supported; only type 0x800", (gint) self->linktype, (gint) eth_type); - return FALSE; + goto done; } b = *buf_ip; /* Check that the packet is IPv4 */ if (((b >> 4) & 0x0f) != 4) - return FALSE; + goto done; ip_header_size = (b & 0x0f) * 4; if (buf_ip + ip_header_size > buf + buf_size) - return FALSE; + goto done; flags = buf_ip[6] >> 5; fragment_offset = @@ -453,7 +452,7 @@ gst_pcap_parse_scan_frame (GstPcapParse * self, GST_LOG_OBJECT (self, "ip proto %d", (gint) ip_protocol); if (ip_protocol != IP_PROTO_UDP && ip_protocol != IP_PROTO_TCP) - return FALSE; + goto done; /* ip info */ ip_src_addr = *((guint32 *) (buf_ip + 12)); @@ -469,36 +468,60 @@ gst_pcap_parse_scan_frame (GstPcapParse * self, if (ip_protocol == IP_PROTO_UDP) { len = GUINT16_FROM_BE (*((guint16 *) (buf_proto + 4))); if (len < UDP_HEADER_LEN || buf_proto + len > buf + buf_size) - return FALSE; + goto done; *payload = buf_proto + UDP_HEADER_LEN; *payload_size = len - UDP_HEADER_LEN; } else { if (buf_proto + 12 >= buf + buf_size) - return FALSE; + goto done; len = (buf_proto[12] >> 4) * 4; if (buf_proto + len > buf + buf_size) - return FALSE; + goto done; /* all remaining data following tcp header is payload */ *payload = buf_proto + len; *payload_size = ip_packet_len - ip_header_size - len; } + src_ip = get_ip_address_as_string (ip_src_addr); + dst_ip = get_ip_address_as_string (ip_dst_addr); + + gst_pcap_parse_add_stats (self, *payload, *payload_size, + src_ip, src_port, dst_ip, dst_port); + /* but still filter as configured */ - if (self->src_ip >= 0 && ip_src_addr != self->src_ip) - return FALSE; + if (self->src_ip && !g_str_equal (src_ip, self->src_ip)) { + GST_LOG_OBJECT (self, "Filtering on src-ip (%s != %s)", + src_ip, self->src_ip); + goto done; + } - if (self->dst_ip >= 0 && ip_dst_addr != self->dst_ip) - return FALSE; + if (self->dst_ip && !g_str_equal (dst_ip, self->dst_ip)) { + GST_LOG_OBJECT (self, "Filtering on dst-ip (%s != %s)", + dst_ip, self->dst_ip); + goto done; + } - if (self->src_port >= 0 && src_port != self->src_port) - return FALSE; + if (self->src_port >= 0 && src_port != self->src_port) { + GST_LOG_OBJECT (self, "Filtering on src-port (%d != %d)", + src_port, self->src_port); + goto done; + } - if (self->dst_port >= 0 && dst_port != self->dst_port) - return FALSE; + if (self->dst_port >= 0 && dst_port != self->dst_port) { + GST_LOG_OBJECT (self, "Filtering on dst-port (%d != %d)", + dst_port, self->dst_port); + goto done; + } + + ret = TRUE; - return TRUE; +done: + g_free (src_ip); + g_free (dst_ip); + + return ret; } static GstFlowReturn @@ -560,10 +583,12 @@ gst_pcap_parse_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) self->cur_packet_size - offset - payload_size); if (GST_CLOCK_TIME_IS_VALID (self->cur_ts)) { - if (!GST_CLOCK_TIME_IS_VALID (self->base_ts)) - self->base_ts = self->cur_ts; + if (!GST_CLOCK_TIME_IS_VALID (self->start_ts)) { + self->start_ts = self->cur_ts; + GST_DEBUG_OBJECT (self, "Setting start_ts to %" GST_TIME_FORMAT, + GST_TIME_ARGS (self->start_ts)); + } if (self->offset >= 0) { - self->cur_ts -= self->base_ts; self->cur_ts += self->offset; } } @@ -572,6 +597,15 @@ gst_pcap_parse_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) if (list == NULL) list = gst_buffer_list_new (); + + /* Take control of the discont flag in order to not have one + * discont buffer be split into multiple */ + if (self->discont) + GST_BUFFER_FLAG_SET (out_buf, GST_BUFFER_FLAG_DISCONT); + else + GST_BUFFER_FLAG_UNSET (out_buf, GST_BUFFER_FLAG_DISCONT); + self->discont = FALSE; + gst_buffer_list_add (list, out_buf); } else { gst_adapter_unmap (self->adapter); @@ -666,13 +700,13 @@ gst_pcap_parse_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) } if (list) { - if (!self->newsegment_sent && GST_CLOCK_TIME_IS_VALID (self->cur_ts)) { + if (!self->newsegment_sent && GST_CLOCK_TIME_IS_VALID (self->start_ts)) { GstSegment segment; if (self->caps) gst_pad_set_caps (self->src_pad, self->caps); gst_segment_init (&segment, GST_FORMAT_TIME); - segment.start = self->base_ts; + gst_segment_set_running_time (&segment, GST_FORMAT_TIME, self->start_ts); gst_pad_push_event (self->src_pad, gst_event_new_segment (&segment)); self->newsegment_sent = TRUE; } @@ -689,6 +723,111 @@ gst_pcap_parse_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) return ret; } +static void +gst_pcap_parse_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstPcapParse *self = GST_PCAP_PARSE (object); + + switch (prop_id) { + case PROP_SRC_IP: + g_value_set_string (value, self->src_ip); + break; + + case PROP_DST_IP: + g_value_set_string (value, self->dst_ip); + break; + + case PROP_SRC_PORT: + g_value_set_int (value, self->src_port); + break; + + case PROP_DST_PORT: + g_value_set_int (value, self->dst_port); + break; + + case PROP_CAPS: + gst_value_set_caps (value, self->caps); + break; + + case PROP_TS_OFFSET: + g_value_set_int64 (value, self->offset); + break; + + case PROP_START_TIME: + g_value_set_int64 (value, self->start_ts); + break; + + case PROP_STATS: + g_value_take_boxed (value, gst_pcap_parse_get_stats (self)); + break; + + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_pcap_parse_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstPcapParse *self = GST_PCAP_PARSE (object); + + switch (prop_id) { + case PROP_SRC_IP: + g_free (self->src_ip); + self->src_ip = g_strdup (g_value_get_string (value)); + break; + + case PROP_DST_IP: + g_free (self->dst_ip); + self->dst_ip = g_strdup (g_value_get_string (value)); + break; + + case PROP_SRC_PORT: + self->src_port = g_value_get_int (value); + break; + + case PROP_DST_PORT: + self->dst_port = g_value_get_int (value); + break; + + case PROP_CAPS: + { + const GstCaps *new_caps_val; + GstCaps *new_caps, *old_caps; + + new_caps_val = gst_value_get_caps (value); + if (new_caps_val == NULL) { + new_caps = gst_caps_new_any (); + } else { + new_caps = gst_caps_copy (new_caps_val); + } + + old_caps = self->caps; + self->caps = new_caps; + if (old_caps) + gst_caps_unref (old_caps); + + gst_pad_set_caps (self->src_pad, new_caps); + break; + } + + case PROP_TS_OFFSET: + self->offset = g_value_get_int64 (value); + break; + + case PROP_START_TIME: + self->start_ts = g_value_get_int64 (value); + break; + + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + static gboolean gst_pcap_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) { @@ -721,13 +860,120 @@ gst_pcap_parse_change_state (GstElement * element, GstStateChange transition) ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); switch (transition) { - case GST_STATE_CHANGE_PAUSED_TO_READY: + case GST_STATE_CHANGE_NULL_TO_READY: gst_pcap_parse_reset (self); break; default: break; } - return ret; } + +static void +gst_pcap_parse_finalize (GObject * object) +{ + GstPcapParse *self = GST_PCAP_PARSE (object); + + g_object_unref (self->adapter); + + /* to get a stats-summary in the debug-log */ + g_value_array_free (gst_pcap_parse_get_stats (self)); + g_hash_table_destroy (self->stats_map); + + g_free (self->src_ip); + g_free (self->dst_ip); + + if (self->caps) + gst_caps_unref (self->caps); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_pcap_parse_class_init (GstPcapParseClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + + gobject_class->finalize = gst_pcap_parse_finalize; + gobject_class->get_property = gst_pcap_parse_get_property; + gobject_class->set_property = gst_pcap_parse_set_property; + + g_object_class_install_property (gobject_class, + PROP_SRC_IP, g_param_spec_string ("src-ip", "Source IP", + "Source IP to restrict to", "", + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_DST_IP, g_param_spec_string ("dst-ip", "Destination IP", + "Destination IP to restrict to", "", + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_SRC_PORT, g_param_spec_int ("src-port", "Source port", + "Source port to restrict to", -1, G_MAXUINT16, -1, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_DST_PORT, g_param_spec_int ("dst-port", "Destination port", + "Destination port to restrict to", -1, G_MAXUINT16, -1, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_CAPS, + g_param_spec_boxed ("caps", "Caps", + "The caps of the source pad", GST_TYPE_CAPS, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_TS_OFFSET, + g_param_spec_int64 ("ts-offset", "Timestamp Offset", + "Relative timestamp offset (ns) to apply (-1 = use absolute packet time)", + -1, G_MAXINT64, -1, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_START_TIME, + g_param_spec_int64 ("start-time", "Start Time", + "The start-time (ns) in the segment (-1 = use first timestamp)", + -1, G_MAXINT64, -1, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_STATS, + g_param_spec_boxed ("stats", "Stats", + "Some stats for the different streams parsed", G_TYPE_VALUE_ARRAY, + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + + gst_element_class_add_static_pad_template (element_class, &sink_template); + gst_element_class_add_static_pad_template (element_class, &src_template); + + element_class->change_state = gst_pcap_parse_change_state; + + gst_element_class_set_static_metadata (element_class, "PCapParse", + "Raw/Parser", + "Parses a raw pcap stream", + "Ole André Vadla Ravnås "); + + GST_DEBUG_CATEGORY_INIT (gst_pcap_parse_debug, "pcapparse", 0, "pcap parser"); +} + +static void +gst_pcap_parse_init (GstPcapParse * self) +{ + self->sink_pad = gst_pad_new_from_static_template (&sink_template, "sink"); + gst_pad_set_chain_function (self->sink_pad, + GST_DEBUG_FUNCPTR (gst_pcap_parse_chain)); + gst_pad_use_fixed_caps (self->sink_pad); + gst_pad_set_event_function (self->sink_pad, + GST_DEBUG_FUNCPTR (gst_pcap_sink_event)); + gst_element_add_pad (GST_ELEMENT (self), self->sink_pad); + + self->src_pad = gst_pad_new_from_static_template (&src_template, "src"); + gst_pad_use_fixed_caps (self->src_pad); + gst_element_add_pad (GST_ELEMENT (self), self->src_pad); + + self->src_port = -1; + self->dst_port = -1; + self->offset = -1; + + self->adapter = gst_adapter_new (); + self->stats_map = + g_hash_table_new_full (g_str_hash, g_str_equal, g_free, + (GDestroyNotify) gst_structure_free); +} diff --git a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.h b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.h index 9547ce4003e..7e866bcfaf3 100644 --- a/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.h +++ b/subprojects/gst-plugins-bad/gst/pcapparse/gstpcapparse.h @@ -39,56 +39,6 @@ G_BEGIN_DECLS typedef struct _GstPcapParse GstPcapParse; typedef struct _GstPcapParseClass GstPcapParseClass; -typedef enum -{ - PCAP_PARSE_STATE_CREATED, - PCAP_PARSE_STATE_PARSING, -} GstPcapParseState; - -typedef enum -{ - LINKTYPE_ETHER = 1, - LINKTYPE_RAW = 101, - LINKTYPE_SLL = 113, - LINKTYPE_SLL2 = 276 -} GstPcapParseLinktype; - -/** - * GstPcapParse: - * - * GstPcapParse element. - */ - -struct _GstPcapParse -{ - GstElement element; - - /*< private >*/ - GstPad * sink_pad; - GstPad * src_pad; - - /* properties */ - gint64 src_ip; - gint64 dst_ip; - gint32 src_port; - gint32 dst_port; - GstCaps *caps; - gint64 offset; - - /* state */ - GstAdapter * adapter; - gboolean initialized; - gboolean swap_endian; - gboolean nanosecond_timestamp; - gint64 cur_packet_size; - GstClockTime cur_ts; - GstClockTime base_ts; - GstPcapParseLinktype linktype; - - gboolean newsegment_sent; - gboolean first_packet; -}; - struct _GstPcapParseClass { GstElementClass parent_class; diff --git a/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c b/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c index 3b44beda4ee..fa2428f6e42 100644 --- a/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c +++ b/subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c @@ -52,7 +52,8 @@ enum GST_H264_PARSE_FORMAT_NONE, GST_H264_PARSE_FORMAT_AVC, GST_H264_PARSE_FORMAT_BYTE, - GST_H264_PARSE_FORMAT_AVC3 + GST_H264_PARSE_FORMAT_AVC3, + GST_H264_PARSE_FORMAT_NALU }; enum @@ -95,8 +96,9 @@ static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_ALWAYS, GST_STATIC_CAPS ("video/x-h264, parsed = (boolean) true, " "stream-format=(string) { avc, avc3, byte-stream }, " - "alignment=(string) { au, nal }")); - + "alignment=(string) { au, nal }; " + "video/x-h264, parsed = (boolean) true, " + "stream-format=(string) nalu-stream, alignment=(string) nal")); #define parent_class gst_h264_parse_parent_class G_DEFINE_TYPE (GstH264Parse, gst_h264_parse, GST_TYPE_BASE_PARSE); GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (h264parse, "h264parse", @@ -362,6 +364,8 @@ gst_h264_parse_get_string (GstH264Parse * parse, gboolean format, gint code) return "byte-stream"; case GST_H264_PARSE_FORMAT_AVC3: return "avc3"; + case GST_H264_PARSE_FORMAT_NALU: + return "nalu-stream"; default: return "none"; } @@ -402,6 +406,8 @@ gst_h264_parse_format_from_caps (GstCaps * caps, guint * format, guint * align) *format = GST_H264_PARSE_FORMAT_BYTE; else if (strcmp (str, "avc3") == 0) *format = GST_H264_PARSE_FORMAT_AVC3; + else if (strcmp (str, "nalu-stream") == 0) + *format = GST_H264_PARSE_FORMAT_NALU; } } @@ -462,6 +468,8 @@ gst_h264_parse_negotiate (GstH264Parse * h264parse, gint in_format, format = GST_H264_PARSE_FORMAT_BYTE; if (!align) align = GST_H264_PARSE_ALIGN_AU; + if (format == GST_H264_PARSE_FORMAT_NALU) + align = GST_H264_PARSE_ALIGN_NAL; GST_DEBUG_OBJECT (h264parse, "selected format %s, alignment %s", gst_h264_parse_get_string (h264parse, TRUE, format), @@ -470,8 +478,8 @@ gst_h264_parse_negotiate (GstH264Parse * h264parse, gint in_format, h264parse->format = format; h264parse->align = align; - h264parse->transform = in_format != h264parse->format || - align == GST_H264_PARSE_ALIGN_AU; + h264parse->transform = (in_format != h264parse->format || + align == GST_H264_PARSE_ALIGN_AU) && h264parse->format != GST_H264_PARSE_FORMAT_NALU; if (caps) gst_caps_unref (caps); @@ -491,12 +499,14 @@ gst_h264_parse_wrap_nal (GstH264Parse * h264parse, guint format, guint8 * data, if (format == GST_H264_PARSE_FORMAT_AVC || format == GST_H264_PARSE_FORMAT_AVC3) { tmp = GUINT32_TO_BE (size << (32 - 8 * nl)); - } else { + } else if (format == GST_H264_PARSE_FORMAT_BYTE) { /* HACK: nl should always be 4 here, otherwise this won't work. * There are legit cases where nl in avc stream is 2, but byte-stream * SC is still always 4 bytes. */ nl = 4; tmp = GUINT32_TO_BE (1); + } else { + nl = 0; } gst_buffer_fill (buf, 0, &tmp, sizeof (guint32)); @@ -1292,6 +1302,12 @@ gst_h264_parse_handle_frame_packetized (GstBaseParse * parse, * Real data is either taken from input by baseclass or * a replacement output buffer is provided anyway. */ gst_h264_parse_parse_frame (parse, &tmp_frame); + + if (h264parse->format == GST_H264_PARSE_FORMAT_NALU) { + g_assert (tmp_frame.out_buffer == NULL); + tmp_frame.out_buffer = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_ALL, nalu.offset, nalu.size); + gst_buffer_copy_into (tmp_frame.out_buffer, tmp_frame.buffer, GST_BUFFER_COPY_METADATA, 0, -1); + } ret = gst_base_parse_finish_frame (parse, &tmp_frame, nl + nalu.size); left -= nl + nalu.size; } @@ -1585,6 +1601,12 @@ gst_h264_parse_handle_frame (GstBaseParse * parse, gst_h264_parse_parse_frame (parse, frame); + if (h264parse->format == GST_H264_PARSE_FORMAT_NALU) { + g_assert (frame->out_buffer == NULL); + frame->out_buffer = gst_buffer_copy_region (frame->buffer, GST_BUFFER_COPY_ALL, nalu.offset, nalu.size); + gst_buffer_copy_into (frame->out_buffer, frame->buffer, GST_BUFFER_COPY_METADATA, 0, -1); + } + return gst_base_parse_finish_frame (parse, frame, framesize); more: @@ -3584,7 +3606,7 @@ gst_h264_parse_set_caps (GstBaseParse * parse, GstCaps * caps) } /* packetized video has codec_data (required for AVC, optional for AVC3) */ - if (codec_data_value != NULL) { + if (format != GST_H264_PARSE_FORMAT_NALU && codec_data_value != NULL) { GstMapInfo map; guint i; @@ -3637,7 +3659,7 @@ gst_h264_parse_set_caps (GstBaseParse * parse, GstCaps * caps) /* don't confuse codec_data with inband sps/pps */ h264parse->have_sps_in_frame = FALSE; h264parse->have_pps_in_frame = FALSE; - } else if (format == GST_H264_PARSE_FORMAT_BYTE) { + } else if (format == GST_H264_PARSE_FORMAT_BYTE || format == GST_H264_PARSE_FORMAT_NALU) { GST_DEBUG_OBJECT (h264parse, "have bytestream h264"); /* nothing to pre-process */ h264parse->packetized = FALSE; diff --git a/subprojects/gst-plugins-bad/meson.build b/subprojects/gst-plugins-bad/meson.build index 9852206ec49..5575207c734 100644 --- a/subprojects/gst-plugins-bad/meson.build +++ b/subprojects/gst-plugins-bad/meson.build @@ -73,7 +73,6 @@ if cc.get_id() == 'msvc' msvc_args += cc.get_supported_arguments([ '/we4002', # too many actual parameters for macro 'identifier' '/we4003', # not enough actual parameters for macro 'identifier' - '/we4013', # 'function' undefined; assuming extern returning int '/we4020', # 'function' : too many actual parameters '/we4027', # function declared without formal parameter list '/we4029', # declared formal parameter list different from definition @@ -83,9 +82,17 @@ if cc.get_id() == 'msvc' '/we4053', # one void operand for '?:' '/we4062', # enumerator 'identifier' in switch of enum 'enumeration' is not handled '/we4098', # 'function' : void function returning a value + ]) + + # add these warnings only if checks are enabled, + # if the checks are not built, it will leave behind some "unused" variables + if get_option('glib-checks').enabled() + msvc_args += cc.get_supported_arguments([ + '/we4013', # 'function' undefined; assuming extern returning int '/we4101', # 'identifier' : unreferenced local variable '/we4189', # 'identifier' : local variable is initialized but not referenced - ]) + ]) + endif endif add_project_arguments(msvc_args, language: ['c', 'cpp']) # Disable SAFESEH with MSVC for plugins and libs that use external deps that diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.c new file mode 100644 index 00000000000..a0a564b2f89 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.c @@ -0,0 +1,457 @@ +/* Gstreamer + * Copyright (C) 2022 Pexip (https://pexip.com/) + * @author: Tulio Beloqui + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "gstacamdeviceprovider.h" + +#include + +struct _GstAcamDeviceProvider +{ + GstDeviceProvider parent; + + ACameraManager *manager; + + struct ACameraManager_AvailabilityListener listeners; +}; + +struct _GstAcamDevice +{ + GstDevice parent; + + gchar *camera_id; + gint camera_index; +}; + +G_DEFINE_TYPE (GstAcamDeviceProvider, gst_acam_device_provider, + GST_TYPE_DEVICE_PROVIDER); + +G_DEFINE_TYPE (GstAcamDevice, gst_acam_device, GST_TYPE_DEVICE); + +GST_DEVICE_PROVIDER_REGISTER_DEFINE (acamdeviceprovider, "acamdeviceprovider", + GST_RANK_PRIMARY, GST_TYPE_ACAM_DEVICE_PROVIDER); + +GST_DEBUG_CATEGORY_STATIC (gst_acam_device_provider_debug); +#define GST_CAT_DEFAULT (gst_acam_device_provider_debug) +#define device_provider_parent_class gst_acam_device_provider_parent_class + +#define CAM_FRONT "Front Camera" +#define CAM_BACK "Back Camera" +#define CAM_EXT "External Camera" +#define CAM_UNKNOWN "Unknown Camera" + + +static gchar * +_build_camera_display_name (gint camera_index, uint8_t lens_facing, + gfloat lens_aperture) +{ + if (camera_index == -1) + return g_strdup (CAM_UNKNOWN); + + if (lens_facing == ACAMERA_LENS_FACING_FRONT) + return g_strdup_printf ("%s (f/%.2f)", CAM_FRONT, lens_aperture); + if (lens_facing == ACAMERA_LENS_FACING_BACK) + return g_strdup_printf ("%s (f/%.2f)", CAM_BACK, lens_aperture); + if (lens_facing == ACAMERA_LENS_FACING_EXTERNAL) + return g_strdup_printf ("%s (%d, f/%.2f)", CAM_EXT, camera_index + 1, + lens_aperture); + + return g_strdup (CAM_UNKNOWN); +} + +static gchar * +gst_acam_device_provider_get_device_display_name (GstAcamDeviceProvider * + provider, const char *camera_id, gint camera_index) +{ + ACameraMetadata *metadata = NULL; + ACameraMetadata_const_entry entry; + uint8_t lens_facing = 0; + gfloat lens_aperture = 0.f; + + if (ACameraManager_getCameraCharacteristics (provider->manager, + camera_id, &metadata) != ACAMERA_OK) { + + GST_ERROR ("Failed to retrieve camera (%s) metadata", camera_id); + return _build_camera_display_name (-1, 0, lens_aperture); + } + + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_LENS_FACING, &entry) != ACAMERA_OK) { + GST_ERROR ("Failed to retrieve camera (%s) LENS_FACING", camera_id); + camera_index = -1; + } else { + lens_facing = entry.data.u8[0]; + } + + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_LENS_INFO_AVAILABLE_APERTURES, &entry) != ACAMERA_OK) { + GST_ERROR + ("Failed to retrieve camera (%s) LENS_INFO_AVAILABLE_FOCAL_LENGTHS", + camera_id); + } else { + lens_aperture = entry.data.f[0]; + } + + ACameraMetadata_free (metadata); + return _build_camera_display_name (camera_index, lens_facing, lens_aperture); +} + +static GstDevice * +gst_acam_device_provider_create_device (GstAcamDeviceProvider * + provider, const char *camera_id, gint camera_index) +{ + GstAcamDevice *device; + gchar *display_name; + GstCaps *caps; + + GST_TRACE_OBJECT (provider, + "Creating device with camera_id: %s, camera_index: %d", camera_id, + camera_index); + + display_name = + gst_acam_device_provider_get_device_display_name (provider, camera_id, + camera_index); + caps = gst_caps_new_empty_simple ("video/x-raw"); + + device = g_object_new (GST_TYPE_ACAM_DEVICE, // + "device-class", "Video/Source", // + "display-name", display_name, // + "caps", caps, // + "camera-id", g_strdup (camera_id), // + "camera-index", camera_index, // + NULL); + + return GST_DEVICE_CAST (device); +} + +static GList * +gst_acam_device_provider_probe (GstDeviceProvider * object) +{ + GList *devices = NULL; + GstAcamDeviceProvider *provider = GST_ACAM_DEVICE_PROVIDER_CAST (object); + ACameraIdList *id_list; + gint i; + + if (ACameraManager_getCameraIdList (provider->manager, + &id_list) != ACAMERA_OK) { + GST_ERROR_OBJECT (provider, "Failed to get camera id list"); + return NULL; + } + + GST_DEBUG_OBJECT (provider, "Found %d cameras", id_list->numCameras); + + for (i = 0; i < id_list->numCameras; i++) { + const char *camera_id = id_list->cameraIds[i]; + GstDevice *device; + + device = gst_acam_device_provider_create_device (provider, camera_id, i); + + gst_object_ref_sink (device); + devices = g_list_append (devices, device); + } + + ACameraManager_deleteCameraIdList (id_list); + return devices; +} + +static void +gst_acam_device_provider_populate_devices (GstDeviceProvider * provider) +{ + GList *devices, *it; + + devices = gst_acam_device_provider_probe (provider); + + for (it = devices; it != NULL; it = g_list_next (it)) { + GstDevice *device = GST_DEVICE_CAST (it->data); + if (device) + gst_device_provider_device_add (provider, device); + } + + g_list_free_full (devices, gst_object_unref); +} + +static gboolean +gst_acam_device_provider_start (GstDeviceProvider * object) +{ + GstAcamDeviceProvider *provider = GST_ACAM_DEVICE_PROVIDER_CAST (object); + + GST_DEBUG_OBJECT (provider, "Starting..."); + + if (!provider->manager) { + GST_ERROR_OBJECT (provider, + "Starting without an instance of ACameraManager"); + return FALSE; + } + + gst_acam_device_provider_populate_devices (object); + + if (ACameraManager_registerAvailabilityCallback (provider->manager, + &provider->listeners) != ACAMERA_OK) { + GST_ERROR_OBJECT (provider, "Couldn't register Availability Callbacks"); + return FALSE; + } + + return TRUE; +} + +static void +gst_acam_device_provider_stop (GstDeviceProvider * object) +{ + GstAcamDeviceProvider *provider = GST_ACAM_DEVICE_PROVIDER_CAST (object); + + GST_DEBUG_OBJECT (provider, "Stopping..."); + + if (!provider->manager) { + GST_ERROR_OBJECT (provider, + "Stopping without an instance of ACameraManager"); + return; + } + + if (ACameraManager_unregisterAvailabilityCallback (provider->manager, + &provider->listeners) != ACAMERA_OK) { + GST_ERROR_OBJECT (provider, "Couldn't unregister Availability Callbacks"); + return; + } +} + +static gint +gst_acam_device_provider_find_camera_index (GstAcamDeviceProvider * + provider, const char *camera_id) +{ + gint i, idx = -1; + ACameraIdList *id_list; + + if (ACameraManager_getCameraIdList (provider->manager, + &id_list) != ACAMERA_OK) { + GST_ERROR_OBJECT (provider, "Failed to get camera id list"); + return -1; + } + + for (i = 0; i < id_list->numCameras; i++) { + const char *id = id_list->cameraIds[i]; + if (g_strcmp0 (camera_id, id) == 0) { + idx = i; + break; + } + } + + ACameraManager_deleteCameraIdList (id_list); + return idx; +} + +static void +gst_acam_device_provider_on_camera_available (void *context, + const char *camera_id) +{ + GstAcamDeviceProvider *provider = GST_ACAM_DEVICE_PROVIDER_CAST (context); + gint camera_index; + GstDevice *device; + + camera_index = + gst_acam_device_provider_find_camera_index (provider, camera_id); + device = + gst_acam_device_provider_create_device (provider, camera_id, + camera_index); + + GST_INFO_OBJECT (provider, "Camera became available (%s)", camera_id); + gst_device_provider_device_add (GST_DEVICE_PROVIDER_CAST (provider), device); +} + +static void +gst_acam_device_provider_remove_device (GstAcamDeviceProvider * acamprovider, + const char *camera_id_to_remove) +{ + GstDeviceProvider *provider = GST_DEVICE_PROVIDER_CAST (acamprovider); + GstDevice *device = NULL; + GList *item; + + GST_OBJECT_LOCK (provider); + for (item = provider->devices; item; item = item->next) { + device = item->data; + gchar *camera_id; + gboolean found; + + g_object_get (device, "camera-id", &camera_id, NULL); + found = (g_strcmp0 (camera_id_to_remove, camera_id) == 0); + g_free (camera_id); + + if (found) { + gst_object_ref (device); + break; + } + + device = NULL; + } + GST_OBJECT_UNLOCK (provider); + + if (device) { + gst_device_provider_device_remove (provider, device); + g_object_unref (device); + } +} + + +static void +gst_acam_device_provider_on_camera_unavailable (void *context, + const char *camera_id) +{ + GstAcamDeviceProvider *provider = GST_ACAM_DEVICE_PROVIDER_CAST (context); + ACameraMetadata *metadata = NULL; + + // if we can't retrieve the metadata for the camera, it has been disconnected + if (ACameraManager_getCameraCharacteristics (provider->manager, + camera_id, &metadata) != ACAMERA_OK) { + GST_INFO_OBJECT (provider, "Camera became unavailable (%s)", camera_id); + gst_acam_device_provider_remove_device (provider, camera_id); + } + + if (metadata) + ACameraMetadata_free (metadata); +} + +static void +gst_acam_device_provider_init (GstAcamDeviceProvider * provider) +{ + provider->manager = ACameraManager_create (); + + provider->listeners.onCameraAvailable = + gst_acam_device_provider_on_camera_available; + provider->listeners.onCameraUnavailable = + gst_acam_device_provider_on_camera_unavailable; + provider->listeners.context = provider; +} + +static void +gst_acam_device_provider_finalize (GObject * object) +{ + GstAcamDeviceProvider *provider = GST_ACAM_DEVICE_PROVIDER_CAST (object); + + if (provider->manager) { + ACameraManager_delete (provider->manager); + provider->manager = NULL; + } + + G_OBJECT_CLASS (device_provider_parent_class)->finalize (object); +} + +static void +gst_acam_device_provider_class_init (GstAcamDeviceProviderClass * klass) +{ + GstDeviceProviderClass *dm_class = GST_DEVICE_PROVIDER_CLASS (klass); + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + dm_class->probe = gst_acam_device_provider_probe; + dm_class->start = gst_acam_device_provider_start; + dm_class->stop = gst_acam_device_provider_stop; + + gobject_class->finalize = gst_acam_device_provider_finalize; + + gst_device_provider_class_set_static_metadata (dm_class, + "Android Camera Device Provider", "Source/Video", + "List and monitor Android camera devices", + "Tulio Beloqui "); + + GST_DEBUG_CATEGORY_INIT (gst_acam_device_provider_debug, + "acamdeviceprovider", 0, "Android Camera Device Provider"); +} + +static GstElement * +gst_acam_device_create_element (GstDevice * object, const gchar * name) +{ + GstAcamDevice *device = GST_ACAM_DEVICE_CAST (object); + GstElement *elem; + + elem = gst_element_factory_make ("ahc2src", name); + g_object_set (elem, "camera-index", device->camera_index, NULL); + + return elem; +} + +enum +{ + PROP_CAMERA_INDEX = 1, + PROP_CAMERA_ID = 2, +}; + +static void +gst_acam_device_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstAcamDevice *device = GST_ACAM_DEVICE_CAST (object); + + switch (prop_id) { + case PROP_CAMERA_INDEX: + g_value_set_int (value, device->camera_index); + break; + case PROP_CAMERA_ID: + g_value_set_string (value, (gchar *) device->camera_id); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_acam_device_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstAcamDevice *device = GST_ACAM_DEVICE_CAST (object); + + switch (prop_id) { + case PROP_CAMERA_INDEX: + device->camera_index = g_value_get_int (value); + break; + case PROP_CAMERA_ID: + if (device->camera_id) + g_free (device->camera_id); + + device->camera_id = g_strdup (g_value_get_string (value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_acam_device_class_init (GstAcamDeviceClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstDeviceClass *dev_class = GST_DEVICE_CLASS (klass); + + dev_class->create_element = gst_acam_device_create_element; + gobject_class->get_property = gst_acam_device_get_property; + gobject_class->set_property = gst_acam_device_set_property; + + g_object_class_install_property (gobject_class, PROP_CAMERA_INDEX, + g_param_spec_int ("camera-index", "Camera Index", + "The camera device index", 0, G_MAXINT, 0, + G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY)); + + g_object_class_install_property (gobject_class, PROP_CAMERA_ID, + g_param_spec_string ("camera-id", "Camera ID", + "The Unique ID for the camera", + NULL, G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY)); +} + +static void +gst_acam_device_init (GstAcamDevice * device) +{ +} diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.h b/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.h new file mode 100644 index 00000000000..d42c181b21b --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstacamdeviceprovider.h @@ -0,0 +1,44 @@ +/* Gstreamer + * Copyright (C) 2022 Pexip (https://pexip.com/) + * @author: Tulio Beloqui + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_ACAM_DEVICE_PROVIDER_H__ +#define __GST_ACAM_DEVICE_PROVIDER_H__ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_ACAM_DEVICE_PROVIDER (gst_acam_device_provider_get_type ()) +#define GST_ACAM_DEVICE_PROVIDER_CAST(obj) ((GstAcamDeviceProvider *)(obj)) + +#define GST_TYPE_ACAM_DEVICE (gst_acam_device_get_type ()) +#define GST_ACAM_DEVICE_CAST(obj) ((GstAcamDevice *)(obj)) + +G_DECLARE_FINAL_TYPE (GstAcamDevice, gst_acam_device, GST, ACAM_DEVICE, + GstDevice) + +G_DECLARE_FINAL_TYPE (GstAcamDeviceProvider, gst_acam_device_provider, GST, + ACAM_DEVICE_PROVIDER, GstDeviceProvider) + +GST_DEVICE_PROVIDER_REGISTER_DECLARE (acamdeviceprovider); + +G_END_DECLS + +#endif /* __GST_ACAM_DEVICE_PROVIDER_H__ */ diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c new file mode 100644 index 00000000000..eefea40fbc9 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.c @@ -0,0 +1,2324 @@ +/* GStreamer android.hardware.Camera2 Source + * Copyright (C) 2017, Collabora Ltd. + * Author:Justin Kim + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "gstahc2src.h" + +#include +#include +#include +#include + +#define GST_USE_UNSTABLE_API +#include + +#include +/* FIXME: NdkCameraManager.h uses 'ACameraDevice_StateCallbacks' + * instead of a small 's'.*/ +#ifndef ACameraDevice_StateCallbacks +#define ACameraDevice_StateCallbacks ACameraDevice_stateCallbacks +#endif +#include + +#include +#include + +#define DEFAULT_MAX_IMAGES 5 + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +GST_DEBUG_CATEGORY_STATIC (gst_debug_ahc2_src); +#define GST_CAT_DEFAULT (gst_debug_ahc2_src) + +struct _GstAHC2Src +{ + GstPushSrc parent; + + GstDataQueue *outbound_queue; + GstClockTime previous_ts; + gboolean started; + gint max_images; + + gint width; + gint height; + + volatile gint referenced_images; + + gint camera_index; + ACameraDevice *camera_device; + ACameraManager *camera_manager; + ACameraIdList *camera_id_list; + gint camera_template_type; + ACameraOutputTarget *camera_output_target; + ACameraCaptureSession *camera_capture_session; + + AImageReader *image_reader; + AImageReader_ImageListener image_reader_listener; + + ACaptureRequest *capture_request; + ACaptureSessionOutput *capture_sout; + ACaptureSessionOutputContainer *capture_sout_container; + + ACameraDevice_stateCallbacks device_state_cb; + ACameraCaptureSession_stateCallbacks session_state_cb; +}; + +typedef struct +{ + volatile gint refcount; + GstAHC2Src *ahc2src; + AImage *image; +} GstWrappedAImage; + +#define GST_TYPE_WRAPPED_AIMAGE (gst_wrapped_aimage_get_type ()) +GType gst_wrapped_aimage_get_type (void); + +static GstWrappedAImage * +gst_wrapped_aimage_ref (GstWrappedAImage * self) +{ + g_return_val_if_fail (self != NULL, NULL); + g_return_val_if_fail (self->ahc2src != NULL, NULL); + g_return_val_if_fail (self->image != NULL, NULL); + g_return_val_if_fail (self->refcount >= 1, NULL); + + GST_TRACE_OBJECT (self->ahc2src, + "ref GstWrappedAImage %p, refcount: %d", self, self->refcount); + + g_atomic_int_add (&self->refcount, 1); + return self; +} + +static void +gst_wrapped_aimage_unref (GstWrappedAImage * self) +{ + g_return_if_fail (self != NULL); + g_return_if_fail (self->ahc2src != NULL); + g_return_if_fail (self->image != NULL); + g_return_if_fail (self->refcount >= 1); + + GST_TRACE_OBJECT (self->ahc2src, + "unref GstWrappedAImage %p, refcount: %d", self, self->refcount); + gint old_ref = g_atomic_int_add (&self->refcount, -1); + + if (old_ref == 1) { + g_atomic_int_add (&self->ahc2src->referenced_images, -1); + GST_TRACE_OBJECT (self->ahc2src, + "Destroying image:%p, referenced_images: %d", self->image, + self->ahc2src->referenced_images); + AImage_delete (self->image); + gst_object_unref (self->ahc2src); + } +} + +G_DEFINE_BOXED_TYPE (GstWrappedAImage, gst_wrapped_aimage, + gst_wrapped_aimage_ref, gst_wrapped_aimage_unref); + +static void gst_ahc2_src_photography_init (gpointer g_iface, + gpointer iface_data); + +#define gst_ahc2_src_parent_class parent_class + +G_DEFINE_TYPE_WITH_CODE (GstAHC2Src, gst_ahc2_src, GST_TYPE_PUSH_SRC, + G_IMPLEMENT_INTERFACE (GST_TYPE_PHOTOGRAPHY, + gst_ahc2_src_photography_init)); + +typedef enum +{ + PROP_CAMERA_INDEX = 1, + PROP_N_CAMERAS, + PROP_CAMERA_TEMPLATE_TYPE, + PROP_MAX_IMAGES, + + /* override photography properties */ + PROP_ANALOG_GAIN, + PROP_APERTURE, + PROP_CAPABILITIES, + PROP_COLOR_TEMPERATURE, + PROP_COLOR_TONE_MODE, + PROP_EV_COMPENSATION, + PROP_EXPOSURE_TIME, + PROP_FLASH_MODE, + PROP_FLICKER_MODE, + PROP_FOCUS_MODE, + PROP_IMAGE_CAPTURE_SUPPORTED_CAPS, + PROP_IMAGE_PREVIEW_SUPPORTED_CAPS, + PROP_ISO_SPEED, + PROP_LENS_FOCUS, + PROP_MAX_EXPOSURE_TIME, + PROP_MIN_EXPOSURE_TIME, + PROP_NOISE_REDUCTION, + PROP_SCENE_MODE, + PROP_WHITE_BALANCE_MODE, + PROP_WHITE_POINT, + PROP_ZOOM, + PROP_EXPOSURE_MODE, + + /*< private > */ + PROP_N_INSTALL = PROP_MAX_IMAGES + 1, + PROP_LAST = PROP_EXPOSURE_MODE, +} GstAHC2SrcProperty; + +static GParamSpec *properties[PROP_LAST + 1]; + +enum +{ + SIG_GET_CAMERA_ID_BY_INDEX, + LAST_SIGNAL +}; + +static guint signals[LAST_SIGNAL] = { 0 }; + +#ifndef IF_EQUAL_RETURN +#define IF_EQUAL_RETURN(param, val) \ + if (val == param) \ + return #val +#endif + +static const gchar * +camera_status_t_to_string (camera_status_t s) +{ + IF_EQUAL_RETURN (s, ACAMERA_OK); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_BASE); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_CAMERA_DEVICE); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_CAMERA_DISABLED); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_CAMERA_DISCONNECTED); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_CAMERA_IN_USE); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_CAMERA_SERVICE); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_INVALID_OPERATION); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_INVALID_PARAMETER); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_MAX_CAMERA_IN_USE); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_METADATA_NOT_FOUND); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_NOT_ENOUGH_MEMORY); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_PERMISSION_DENIED); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_SESSION_CLOSED); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_STREAM_CONFIGURE_FAIL); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_UNKNOWN); + IF_EQUAL_RETURN (s, ACAMERA_ERROR_UNSUPPORTED_OPERATION); + return NULL; +} + +static const gchar * +camera_error_to_string (int e) +{ + IF_EQUAL_RETURN (e, ERROR_CAMERA_IN_USE); + IF_EQUAL_RETURN (e, ERROR_MAX_CAMERAS_IN_USE); + IF_EQUAL_RETURN (e, ERROR_CAMERA_DISABLED); + IF_EQUAL_RETURN (e, ERROR_CAMERA_DEVICE); + IF_EQUAL_RETURN (e, ERROR_CAMERA_SERVICE); + return NULL; +} + +static GstPhotographyCaps +gst_ahc2_src_get_capabilities (GstPhotography * photo) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + + GstPhotographyCaps caps = GST_PHOTOGRAPHY_CAPS_EV_COMP | + GST_PHOTOGRAPHY_CAPS_WB_MODE | GST_PHOTOGRAPHY_CAPS_TONE | + GST_PHOTOGRAPHY_CAPS_SCENE | GST_PHOTOGRAPHY_CAPS_FLASH | + GST_PHOTOGRAPHY_CAPS_FOCUS; + + ACameraMetadata *metadata = NULL; + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->camera_manager != NULL, caps); + g_return_val_if_fail (self->camera_id_list != NULL, caps); + g_return_val_if_fail (self->camera_index < self->camera_id_list->numCameras, + caps); + + if (ACameraManager_getCameraCharacteristics (self->camera_manager, + self->camera_id_list->cameraIds[self->camera_index], + &metadata) != ACAMERA_OK) { + return caps; + } + + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_LENS_INFO_AVAILABLE_FOCAL_LENGTHS, &entry) == ACAMERA_OK) { + if (entry.count > 1) { + caps |= GST_PHOTOGRAPHY_CAPS_ZOOM; + } + } + + ACameraMetadata_free (metadata); + + return caps; +} + +static gboolean +gst_ahc2_src_get_color_tone_mode (GstPhotography * photo, + GstPhotographyColorToneMode * tone_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_CONTROL_EFFECT_MODE, &entry) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get EFFECT MODE"); + return FALSE; + } + + switch (entry.data.u8[0]) { + case ACAMERA_CONTROL_EFFECT_MODE_OFF: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_NORMAL; + break; + case ACAMERA_CONTROL_EFFECT_MODE_MONO: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_GRAYSCALE; + break; + case ACAMERA_CONTROL_EFFECT_MODE_NEGATIVE: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_NEGATIVE; + break; + case ACAMERA_CONTROL_EFFECT_MODE_SOLARIZE: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_SOLARIZE; + break; + case ACAMERA_CONTROL_EFFECT_MODE_SEPIA: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_SEPIA; + break; + case ACAMERA_CONTROL_EFFECT_MODE_POSTERIZE: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_POSTERIZE; + break; + case ACAMERA_CONTROL_EFFECT_MODE_WHITEBOARD: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_WHITEBOARD; + break; + case ACAMERA_CONTROL_EFFECT_MODE_BLACKBOARD: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_BLACKBOARD; + break; + case ACAMERA_CONTROL_EFFECT_MODE_AQUA: + *tone_mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_AQUA; + break; + default: + g_assert_not_reached (); + break; + } + + return TRUE; +} + +static gboolean +gst_ahc2_src_set_color_tone_mode (GstPhotography * photo, + GstPhotographyColorToneMode tone_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + guint8 mode; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + switch (tone_mode) { + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_NORMAL: + mode = ACAMERA_CONTROL_EFFECT_MODE_OFF; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_SEPIA: + mode = ACAMERA_CONTROL_EFFECT_MODE_SEPIA; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_NEGATIVE: + mode = ACAMERA_CONTROL_EFFECT_MODE_NEGATIVE; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_GRAYSCALE: + mode = ACAMERA_CONTROL_EFFECT_MODE_MONO; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_SOLARIZE: + mode = ACAMERA_CONTROL_EFFECT_MODE_SOLARIZE; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_POSTERIZE: + mode = GST_PHOTOGRAPHY_COLOR_TONE_MODE_POSTERIZE; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_WHITEBOARD: + mode = ACAMERA_CONTROL_EFFECT_MODE_WHITEBOARD; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_BLACKBOARD: + mode = ACAMERA_CONTROL_EFFECT_MODE_BLACKBOARD; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_AQUA: + mode = ACAMERA_CONTROL_EFFECT_MODE_AQUA; + break; + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_NATURAL: + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_VIVID: + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_COLORSWAP: + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_OUT_OF_FOCUS: + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_SKY_BLUE: + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_GRASS_GREEN: + case GST_PHOTOGRAPHY_COLOR_TONE_MODE_SKIN_WHITEN: + default: + GST_WARNING_OBJECT (self, "Unsupported color tone (%d)", tone_mode); + return FALSE; + } + + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_u8 (self->capture_request, + ACAMERA_CONTROL_EFFECT_MODE, 1, &mode); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + return TRUE; +} + +static void +gst_ahc2_src_set_autofocus (GstPhotography * photo, gboolean on) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + + guint8 mode = on ? ACAMERA_CONTROL_AF_MODE_AUTO : ACAMERA_CONTROL_AF_MODE_OFF; + + g_return_if_fail (self->capture_request != NULL); + + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_u8 (self->capture_request, + ACAMERA_CONTROL_AF_MODE, 1, &mode); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); +} + +static gboolean +gst_ahc2_src_get_focus_mode (GstPhotography * photo, + GstPhotographyFocusMode * focus_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_CONTROL_AF_MODE, &entry) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get AF MODE"); + return FALSE; + } + + /* GstPhotographyFocusMode doesn't matches + * acamera_metadata_enum_acamera_control_af_mode. */ + switch (entry.data.u8[0]) { + case ACAMERA_CONTROL_AF_MODE_OFF: + *focus_mode = GST_PHOTOGRAPHY_FOCUS_MODE_MANUAL; + break; + case ACAMERA_CONTROL_AF_MODE_AUTO: + *focus_mode = GST_PHOTOGRAPHY_FOCUS_MODE_AUTO; + break; + case ACAMERA_CONTROL_AF_MODE_MACRO: + *focus_mode = GST_PHOTOGRAPHY_FOCUS_MODE_MACRO; + break; + case ACAMERA_CONTROL_AF_MODE_CONTINUOUS_VIDEO: + *focus_mode = GST_PHOTOGRAPHY_FOCUS_MODE_HYPERFOCAL; + break; + case ACAMERA_CONTROL_AF_MODE_CONTINUOUS_PICTURE: + *focus_mode = GST_PHOTOGRAPHY_FOCUS_MODE_CONTINUOUS_NORMAL; + break; + case ACAMERA_CONTROL_AF_MODE_EDOF: + *focus_mode = GST_PHOTOGRAPHY_FOCUS_MODE_EXTENDED; + break; + default: + g_assert_not_reached (); + break; + } + + return TRUE; +} + +static gboolean +gst_ahc2_src_set_focus_mode (GstPhotography * photo, + GstPhotographyFocusMode focus_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + + guint8 mode; + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + switch (focus_mode) { + case GST_PHOTOGRAPHY_FOCUS_MODE_AUTO: + mode = ACAMERA_CONTROL_AF_MODE_AUTO; + break; + case GST_PHOTOGRAPHY_FOCUS_MODE_MACRO: + mode = ACAMERA_CONTROL_AF_MODE_MACRO; + break; + case GST_PHOTOGRAPHY_FOCUS_MODE_PORTRAIT: + mode = ACAMERA_CONTROL_AF_MODE_CONTINUOUS_PICTURE; + break; + case GST_PHOTOGRAPHY_FOCUS_MODE_INFINITY: + case GST_PHOTOGRAPHY_FOCUS_MODE_HYPERFOCAL: + case GST_PHOTOGRAPHY_FOCUS_MODE_CONTINUOUS_NORMAL: + case GST_PHOTOGRAPHY_FOCUS_MODE_CONTINUOUS_EXTENDED: + mode = ACAMERA_CONTROL_AF_MODE_CONTINUOUS_VIDEO; + break; + case GST_PHOTOGRAPHY_FOCUS_MODE_MANUAL: + mode = ACAMERA_CONTROL_AF_MODE_OFF; + break; + case GST_PHOTOGRAPHY_FOCUS_MODE_EXTENDED: + mode = ACAMERA_CONTROL_AF_MODE_EDOF; + break; + default: + g_assert_not_reached (); + break; + } + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_u8 (self->capture_request, + ACAMERA_CONTROL_AF_MODE, 1, &mode); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + return TRUE; +} + +static gboolean +gst_ahc2_src_get_exposure_mode (GstPhotography * photo, + GstPhotographyExposureMode * exposure_mode) +{ + (void) photo; + (void) exposure_mode; + return FALSE; +} + +static gboolean +gst_ahc2_src_set_exposure_mode (GstPhotography * photo, + GstPhotographyExposureMode exposure_mode) +{ + (void) photo; + (void) exposure_mode; + return TRUE; +} + +static gboolean +gst_ahc2_src_get_ev_compensation (GstPhotography * photo, gfloat * ev_comp) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + + ACameraMetadata *metadata = NULL; + ACameraMetadata_const_entry entry; + gint ev; + ACameraMetadata_rational ev_step; + gboolean ret = FALSE; + + g_return_val_if_fail (self->camera_manager != NULL, FALSE); + g_return_val_if_fail (self->camera_id_list != NULL, FALSE); + g_return_val_if_fail (self->camera_index < self->camera_id_list->numCameras, + FALSE); + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACameraManager_getCameraCharacteristics (self->camera_manager, + self->camera_id_list->cameraIds[self->camera_index], + &metadata) != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to get metadata object"); + goto out; + } + + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_CONTROL_AE_COMPENSATION_STEP, &entry) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get EV STEP"); + goto out; + } + + ev_step = entry.data.r[0]; + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_CONTROL_AE_EXPOSURE_COMPENSATION, &entry) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get EV COMPENSATION"); + goto out; + } + + ev = entry.data.i32[0]; + + *ev_comp = (gfloat) (ev * ev_step.numerator / ev_step.denominator); + + ret = TRUE; +out: + ACameraMetadata_free (metadata); + return ret; +} + +static gboolean +gst_ahc2_src_set_ev_compensation (GstPhotography * photo, gfloat ev_comp) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + + ACameraMetadata *metadata = NULL; + ACameraMetadata_const_entry entry; + gint ev, ev_min, ev_max; + ACameraMetadata_rational ev_step; + gboolean ret = FALSE; + + g_return_val_if_fail (self->camera_manager != NULL, FALSE); + g_return_val_if_fail (self->camera_id_list != NULL, FALSE); + g_return_val_if_fail (self->camera_index < self->camera_id_list->numCameras, + FALSE); + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACameraManager_getCameraCharacteristics (self->camera_manager, + self->camera_id_list->cameraIds[self->camera_index], + &metadata) != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to get metadata object"); + goto out; + } + + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_CONTROL_AE_COMPENSATION_STEP, &entry) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get EV STEP"); + goto out; + } + + ev_step = entry.data.r[0]; + + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_CONTROL_AE_COMPENSATION_RANGE, &entry) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get EV RANGE"); + goto out; + } + + ev_min = entry.data.i32[0]; + ev_max = entry.data.i32[1]; + + ev = (gint) round (ev_comp / ev_step.numerator * ev_step.denominator); + + if (ev < ev_min) { + GST_WARNING_OBJECT (self, "converted EV(%d) is less than min(%d)", ev, + ev_min); + ev = ev_min; + } + + if (ev > ev_max) { + GST_WARNING_OBJECT (self, "converted EV(%d) is greater than max(%d)", ev, + ev_max); + ev = ev_max; + } + + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_i32 (self->capture_request, + ACAMERA_CONTROL_AE_EXPOSURE_COMPENSATION, 1, &ev); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + ret = TRUE; +out: + ACameraMetadata_free (metadata); + return ret; +} + +static gboolean +gst_ahc2_src_get_flash_mode (GstPhotography * photo, + GstPhotographyFlashMode * flash_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_FLASH_MODE, &entry) != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to get FLASH MODE"); + return FALSE; + } + + switch (entry.data.u8[0]) { + case ACAMERA_FLASH_MODE_OFF: + *flash_mode = GST_PHOTOGRAPHY_FLASH_MODE_OFF; + break; + case ACAMERA_FLASH_MODE_SINGLE: + *flash_mode = GST_PHOTOGRAPHY_FLASH_MODE_AUTO; + break; + case ACAMERA_FLASH_MODE_TORCH: + *flash_mode = GST_PHOTOGRAPHY_FLASH_MODE_ON; + break; + default: + g_assert_not_reached (); + break; + } + + return TRUE; +} + +static gboolean +gst_ahc2_src_set_flash_mode (GstPhotography * photo, + GstPhotographyFlashMode flash_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + guint8 mode; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + switch (flash_mode) { + case GST_PHOTOGRAPHY_FLASH_MODE_OFF: + mode = ACAMERA_FLASH_MODE_OFF; + break; + case GST_PHOTOGRAPHY_FLASH_MODE_ON: + mode = ACAMERA_FLASH_MODE_TORCH; + break; + case GST_PHOTOGRAPHY_FLASH_MODE_AUTO: + case GST_PHOTOGRAPHY_FLASH_MODE_FILL_IN: + case GST_PHOTOGRAPHY_FLASH_MODE_RED_EYE: + mode = ACAMERA_FLASH_MODE_SINGLE; + break; + default: + g_assert_not_reached (); + break; + } + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_u8 (self->capture_request, + ACAMERA_FLASH_MODE, 1, &mode); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + return TRUE; +} + +static gboolean +gst_ahc2_src_get_flicker_mode (GstPhotography * photo, + GstPhotographyFlickerReductionMode * flicker_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_CONTROL_AE_ANTIBANDING_MODE, &entry) != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to get ANTIBANDING MODE"); + return FALSE; + } + + switch (entry.data.u8[0]) { + case ACAMERA_CONTROL_AE_ANTIBANDING_MODE_OFF: + *flicker_mode = GST_PHOTOGRAPHY_FLICKER_REDUCTION_OFF; + break; + case ACAMERA_CONTROL_AE_ANTIBANDING_MODE_50HZ: + *flicker_mode = GST_PHOTOGRAPHY_FLICKER_REDUCTION_50HZ; + break; + case ACAMERA_CONTROL_AE_ANTIBANDING_MODE_60HZ: + *flicker_mode = GST_PHOTOGRAPHY_FLICKER_REDUCTION_60HZ; + break; + case ACAMERA_CONTROL_AE_ANTIBANDING_MODE_AUTO: + *flicker_mode = GST_PHOTOGRAPHY_FLICKER_REDUCTION_AUTO; + break; + default: + g_assert_not_reached (); + break; + } + + return TRUE; +} + +static gboolean +gst_ahc2_src_set_flicker_mode (GstPhotography * photo, + GstPhotographyFlickerReductionMode flicker_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + guint8 mode; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + switch (flicker_mode) { + case GST_PHOTOGRAPHY_FLICKER_REDUCTION_OFF: + mode = ACAMERA_CONTROL_AE_ANTIBANDING_MODE_OFF; + break; + case GST_PHOTOGRAPHY_FLICKER_REDUCTION_50HZ: + mode = ACAMERA_CONTROL_AE_ANTIBANDING_MODE_50HZ; + break; + case GST_PHOTOGRAPHY_FLICKER_REDUCTION_60HZ: + mode = ACAMERA_CONTROL_AE_ANTIBANDING_MODE_60HZ; + break; + case GST_PHOTOGRAPHY_FLICKER_REDUCTION_AUTO: + mode = ACAMERA_CONTROL_AE_ANTIBANDING_MODE_AUTO; + break; + default: + GST_WARNING_OBJECT (self, "Unsupported flicker-mode (%d)", flicker_mode); + break; + } + + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_u8 (self->capture_request, + ACAMERA_CONTROL_AE_ANTIBANDING_MODE, 1, &mode); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + return TRUE; +} + +static gboolean +gst_ahc2_src_get_scene_mode (GstPhotography * photo, + GstPhotographySceneMode * scene_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_CONTROL_SCENE_MODE, &entry) != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to get SCENE MODE"); + return FALSE; + } + + switch (entry.data.u8[0]) { + case ACAMERA_CONTROL_SCENE_MODE_DISABLED: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_MANUAL; + break; + case ACAMERA_CONTROL_SCENE_MODE_ACTION: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_ACTION; + break; + case ACAMERA_CONTROL_SCENE_MODE_PORTRAIT: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_PORTRAIT; + break; + case ACAMERA_CONTROL_SCENE_MODE_LANDSCAPE: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_LANDSCAPE; + break; + case ACAMERA_CONTROL_SCENE_MODE_NIGHT: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_NIGHT; + break; + case ACAMERA_CONTROL_SCENE_MODE_NIGHT_PORTRAIT: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_NIGHT_PORTRAIT; + break; + case ACAMERA_CONTROL_SCENE_MODE_THEATRE: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_THEATRE; + break; + case ACAMERA_CONTROL_SCENE_MODE_BEACH: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_BEACH; + break; + case ACAMERA_CONTROL_SCENE_MODE_SNOW: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_SNOW; + break; + case ACAMERA_CONTROL_SCENE_MODE_SUNSET: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_SUNSET; + break; + case ACAMERA_CONTROL_SCENE_MODE_STEADYPHOTO: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_STEADY_PHOTO; + break; + case ACAMERA_CONTROL_SCENE_MODE_FIREWORKS: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_FIREWORKS; + break; + case ACAMERA_CONTROL_SCENE_MODE_SPORTS: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_SPORT; + break; + case ACAMERA_CONTROL_SCENE_MODE_PARTY: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_PARTY; + break; + case ACAMERA_CONTROL_SCENE_MODE_CANDLELIGHT: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_CANDLELIGHT; + break; + case ACAMERA_CONTROL_SCENE_MODE_BARCODE: + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_BARCODE; + break; + case ACAMERA_CONTROL_SCENE_MODE_HDR: + case ACAMERA_CONTROL_SCENE_MODE_FACE_PRIORITY: + GST_WARNING_OBJECT (self, + "Device returns scene mode (%d), but no proper interface is defined", + entry.data.u8[0]); + *scene_mode = GST_PHOTOGRAPHY_SCENE_MODE_MANUAL; + break; + default: + g_assert_not_reached (); + break; + } + + return TRUE; +} + +static gboolean +gst_ahc2_src_set_scene_mode (GstPhotography * photo, + GstPhotographySceneMode scene_mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + guint8 mode; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + switch (scene_mode) { + case GST_PHOTOGRAPHY_SCENE_MODE_MANUAL: + mode = ACAMERA_CONTROL_SCENE_MODE_DISABLED; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_PORTRAIT: + mode = ACAMERA_CONTROL_SCENE_MODE_PORTRAIT; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_LANDSCAPE: + mode = ACAMERA_CONTROL_SCENE_MODE_LANDSCAPE; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_SPORT: + mode = ACAMERA_CONTROL_SCENE_MODE_SPORTS; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_NIGHT: + mode = ACAMERA_CONTROL_SCENE_MODE_NIGHT; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_ACTION: + mode = ACAMERA_CONTROL_SCENE_MODE_ACTION; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_NIGHT_PORTRAIT: + mode = ACAMERA_CONTROL_SCENE_MODE_NIGHT_PORTRAIT; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_THEATRE: + mode = ACAMERA_CONTROL_SCENE_MODE_THEATRE; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_BEACH: + mode = ACAMERA_CONTROL_SCENE_MODE_BEACH; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_SNOW: + mode = ACAMERA_CONTROL_SCENE_MODE_SNOW; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_SUNSET: + mode = ACAMERA_CONTROL_SCENE_MODE_SUNSET; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_STEADY_PHOTO: + mode = ACAMERA_CONTROL_SCENE_MODE_STEADYPHOTO; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_FIREWORKS: + mode = ACAMERA_CONTROL_SCENE_MODE_FIREWORKS; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_PARTY: + mode = ACAMERA_CONTROL_SCENE_MODE_PARTY; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_CANDLELIGHT: + mode = ACAMERA_CONTROL_SCENE_MODE_CANDLELIGHT; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_BARCODE: + mode = ACAMERA_CONTROL_SCENE_MODE_BARCODE; + break; + case GST_PHOTOGRAPHY_SCENE_MODE_AUTO: + case GST_PHOTOGRAPHY_SCENE_MODE_CLOSEUP: + default: + GST_WARNING_OBJECT (self, "Unsupported scene mode (%d)", scene_mode); + return FALSE; + } + + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_u8 (self->capture_request, + ACAMERA_CONTROL_SCENE_MODE, 1, &mode); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + return TRUE; +} + +static gboolean +gst_ahc2_src_get_white_balance_mode (GstPhotography * photo, + GstPhotographyWhiteBalanceMode * mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_CONTROL_AWB_MODE, &entry) != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to get AWB_MODE"); + return FALSE; + } + + switch (entry.data.u8[0]) { + case ACAMERA_CONTROL_AWB_MODE_OFF: + *mode = GST_PHOTOGRAPHY_WB_MODE_MANUAL; + break; + case ACAMERA_CONTROL_AWB_MODE_AUTO: + *mode = GST_PHOTOGRAPHY_WB_MODE_AUTO; + break; + case ACAMERA_CONTROL_AWB_MODE_INCANDESCENT: + *mode = GST_PHOTOGRAPHY_WB_MODE_TUNGSTEN; + break; + case ACAMERA_CONTROL_AWB_MODE_FLUORESCENT: + *mode = GST_PHOTOGRAPHY_WB_MODE_FLUORESCENT; + break; + case ACAMERA_CONTROL_AWB_MODE_WARM_FLUORESCENT: + *mode = GST_PHOTOGRAPHY_WB_MODE_WARM_FLUORESCENT; + break; + case ACAMERA_CONTROL_AWB_MODE_DAYLIGHT: + *mode = GST_PHOTOGRAPHY_WB_MODE_DAYLIGHT; + break; + case ACAMERA_CONTROL_AWB_MODE_CLOUDY_DAYLIGHT: + *mode = GST_PHOTOGRAPHY_WB_MODE_CLOUDY; + break; + case ACAMERA_CONTROL_AWB_MODE_TWILIGHT: + *mode = GST_PHOTOGRAPHY_WB_MODE_SUNSET; + break; + case ACAMERA_CONTROL_AWB_MODE_SHADE: + *mode = GST_PHOTOGRAPHY_WB_MODE_SHADE; + break; + default: + GST_WARNING_OBJECT (self, "Uknown AWB parameter (0x%x)", + entry.data.u8[0]); + return FALSE; + break; + } + + GST_DEBUG_OBJECT (self, "AWB parameter (0x%x -> 0x%x)", entry.data.u8[0], + *mode); + + return TRUE; +} + +static gboolean +gst_ahc2_src_set_white_balance_mode (GstPhotography * photo, + GstPhotographyWhiteBalanceMode mode) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + guint8 wb_mode; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + switch (mode) { + case GST_PHOTOGRAPHY_WB_MODE_AUTO: + wb_mode = ACAMERA_CONTROL_AWB_MODE_AUTO; + break; + case GST_PHOTOGRAPHY_WB_MODE_DAYLIGHT: + wb_mode = ACAMERA_CONTROL_AWB_MODE_DAYLIGHT; + break; + case GST_PHOTOGRAPHY_WB_MODE_CLOUDY: + wb_mode = ACAMERA_CONTROL_AWB_MODE_CLOUDY_DAYLIGHT; + break; + case GST_PHOTOGRAPHY_WB_MODE_SUNSET: + wb_mode = ACAMERA_CONTROL_AWB_MODE_TWILIGHT; + break; + case GST_PHOTOGRAPHY_WB_MODE_TUNGSTEN: + wb_mode = ACAMERA_CONTROL_AWB_MODE_INCANDESCENT; + break; + case GST_PHOTOGRAPHY_WB_MODE_FLUORESCENT: + wb_mode = ACAMERA_CONTROL_AWB_MODE_FLUORESCENT; + break; + case GST_PHOTOGRAPHY_WB_MODE_MANUAL: + wb_mode = ACAMERA_CONTROL_AWB_MODE_OFF; + break; + case GST_PHOTOGRAPHY_WB_MODE_WARM_FLUORESCENT: + wb_mode = ACAMERA_CONTROL_AWB_MODE_WARM_FLUORESCENT; + break; + case GST_PHOTOGRAPHY_WB_MODE_SHADE: + wb_mode = ACAMERA_CONTROL_AWB_MODE_SHADE; + break; + default: + g_assert_not_reached (); + break; + } + + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + ACaptureRequest_setEntry_u8 (self->capture_request, + ACAMERA_CONTROL_AWB_MODE, 1, &wb_mode); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + return TRUE; +} + +static gboolean +gst_ahc2_src_get_zoom (GstPhotography * photo, gfloat * zoom) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACaptureRequest_getConstEntry (self->capture_request, + ACAMERA_LENS_FOCAL_LENGTH, &entry) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get FOCAL LENGTH (optical zoom)"); + return FALSE; + } + + *zoom = entry.data.f[0]; + + return TRUE; +} + +static gboolean +gst_ahc2_src_set_zoom (GstPhotography * photo, gfloat zoom) +{ + GstAHC2Src *self = GST_AHC2_SRC (photo); + + gboolean ret = FALSE; + ACameraMetadata *metadata = NULL; + ACameraMetadata_const_entry entry; + + g_return_val_if_fail (self->camera_manager != NULL, FALSE); + g_return_val_if_fail (self->camera_id_list != NULL, FALSE); + g_return_val_if_fail (self->camera_index < self->camera_id_list->numCameras, + FALSE); + g_return_val_if_fail (self->capture_request != NULL, FALSE); + + if (ACameraManager_getCameraCharacteristics (self->camera_manager, + self->camera_id_list->cameraIds[self->camera_index], + &metadata) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to get metadata"); + return FALSE; + } + + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_LENS_INFO_AVAILABLE_FOCAL_LENGTHS, &entry) != ACAMERA_OK) { + goto out; + } + + if (entry.count == 1) { + GST_WARNING_OBJECT (self, + "This device has fixed focal length(%.2f).", entry.data.f[0]); + goto out; + } + + ACameraCaptureSession_stopRepeating (self->camera_capture_session); + + /* FIXME: Do we need to set nearest value? */ + ACaptureRequest_setEntry_float (self->capture_request, + ACAMERA_LENS_FOCAL_LENGTH, 1, &zoom); + + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + ret = TRUE; +out: + ACameraMetadata_free (metadata); + + return ret; +} + +static GstCaps * +gst_ahc2_src_fixate (GstBaseSrc * src, GstCaps * caps) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + GstStructure *s = gst_caps_get_structure (caps, 0); + + GST_DEBUG_OBJECT (self, "Fixating : %" GST_PTR_FORMAT, caps); + + caps = gst_caps_make_writable (caps); + + /* Width/height will be fixed already here, format will + * be left for fixation by the default handler. + * We only have to fixate framerate here, to the + * highest possible framerate. + */ + gst_structure_fixate_field_nearest_fraction (s, "framerate", G_MAXINT, 1); + + caps = GST_BASE_SRC_CLASS (parent_class)->fixate (src, caps); + + return caps; +} + +static gboolean +gst_ahc2_src_set_caps (GstBaseSrc * src, GstCaps * caps) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + + GstStructure *s; + GstVideoFormat format; + gint fmt = -1; + const gchar *format_str = NULL; + const GValue *framerate_value; + gint fps_min = 0, fps_max = 0; + + ANativeWindow *image_reader_window = NULL; + + g_return_val_if_fail (self->camera_index < self->camera_id_list->numCameras, + FALSE); + + GST_DEBUG_OBJECT (self, "%" GST_PTR_FORMAT, caps); + + s = gst_caps_get_structure (caps, 0); + format_str = gst_structure_get_string (s, "format"); + format = gst_video_format_from_string (format_str); + + gst_structure_get_int (s, "width", &self->width); + gst_structure_get_int (s, "height", &self->height); + framerate_value = gst_structure_get_value (s, "framerate"); + + if (GST_VALUE_HOLDS_FRACTION_RANGE (framerate_value)) { + const GValue *min, *max; + + min = gst_value_get_fraction_range_min (framerate_value); + max = gst_value_get_fraction_range_max (framerate_value); + + fps_min = + gst_value_get_fraction_numerator (min) / + gst_value_get_fraction_denominator (min); + fps_max = + gst_value_get_fraction_numerator (max) / + gst_value_get_fraction_denominator (max); + + } else if (GST_VALUE_HOLDS_FRACTION (framerate_value)) { + fps_min = fps_max = + gst_value_get_fraction_numerator (framerate_value) / + gst_value_get_fraction_denominator (framerate_value); + } else { + GST_WARNING_OBJECT (self, "framerate holds unrecognizable value"); + } + + GST_OBJECT_LOCK (self); + + if (fps_min != 0 && fps_max != 0) { + gint fps_range[2] = { fps_min, fps_max }; + ACaptureRequest_setEntry_i32 (self->capture_request, + ACAMERA_CONTROL_AE_TARGET_FPS_RANGE, 2, fps_range); + GST_DEBUG_OBJECT (self, "setting fps range [%d, %d]", fps_min, fps_max); + } + + switch (format) { + case GST_VIDEO_FORMAT_NV12: + fmt = AIMAGE_FORMAT_YUV_420_888; + break; + default: + GST_WARNING_OBJECT (self, "unsupported video format (%s)", format_str); + goto failed; + } + + /* we flush the internal queue, so that old AImages can be cleaned out */ + gst_data_queue_flush (self->outbound_queue); + + while (self->referenced_images > 0) { + g_usleep (G_USEC_PER_SEC / 50); + GST_DEBUG_OBJECT (self, + "waiting for buffers to come home. referenced_images: %d", + self->referenced_images); + } + + /* this will possibly invalidate any AImge currently in the pipeline */ + g_clear_pointer (&self->image_reader, (GDestroyNotify) AImageReader_delete); + + if (AImageReader_new (self->width, self->height, fmt, self->max_images, + &self->image_reader) != AMEDIA_OK) { + GST_ERROR_OBJECT (self, "One or more of parameter " + "(width: %d, height: %d, format: %s, max-images: %d) " + "is not supported.", self->width, self->height, format_str, + self->max_images); + + goto failed; + } + + if (AImageReader_getWindow (self->image_reader, + &image_reader_window) != AMEDIA_OK) { + GST_WARNING_OBJECT (self, "Failed to get AImageReader surface"); + goto failed; + } + + AImageReader_setImageListener (self->image_reader, + &self->image_reader_listener); + + if (self->camera_output_target != NULL) { + ACaptureRequest_removeTarget (self->capture_request, + self->camera_output_target); + } + g_clear_pointer (&self->camera_output_target, + (GDestroyNotify) ACameraOutputTarget_free); + if (ACameraOutputTarget_create (image_reader_window, + &self->camera_output_target) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to create ACameraOuputTarget"); + goto failed; + } + + if (self->capture_sout != NULL) { + ACaptureSessionOutputContainer_remove (self->capture_sout_container, + self->capture_sout); + } + g_clear_pointer (&self->capture_sout, + (GDestroyNotify) ACaptureSessionOutput_free); + if (ACaptureSessionOutput_create (image_reader_window, + &self->capture_sout) != ACAMERA_OK) { + GST_WARNING_OBJECT (self, "Failed to create ACaptureSessionOutput"); + goto failed; + } + + ACaptureRequest_addTarget (self->capture_request, self->camera_output_target); + ACaptureSessionOutputContainer_add (self->capture_sout_container, + self->capture_sout); + + ACameraDevice_createCaptureSession (self->camera_device, + self->capture_sout_container, &self->session_state_cb, + &self->camera_capture_session); + + GST_DEBUG_OBJECT (self, "Starting capture request"); + ACameraCaptureSession_setRepeatingRequest (self->camera_capture_session, + NULL, 1, &self->capture_request, NULL); + + GST_OBJECT_UNLOCK (self); + + return TRUE; + +failed: + GST_OBJECT_UNLOCK (self); + + g_clear_pointer (&self->image_reader, (GDestroyNotify) AImageReader_delete); + g_clear_pointer (&self->camera_output_target, + (GDestroyNotify) ACameraOutputTarget_free); + + return FALSE; +} + +static GstCaps * +gst_ahc2_src_get_caps (GstBaseSrc * src, GstCaps * filter) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + GstCaps *caps = gst_caps_new_empty (); + GstStructure *format = NULL; + + ACameraMetadata *metadata = NULL; + ACameraMetadata_const_entry entry, fps_entry; + + GST_OBJECT_LOCK (self); + + if (ACameraManager_getCameraCharacteristics (self->camera_manager, + self->camera_id_list->cameraIds[self->camera_index], + &metadata) != ACAMERA_OK) { + + GST_ERROR_OBJECT (self, "Failed to get metadata from camera device"); + + GST_OBJECT_UNLOCK (self); + return caps; + } +#ifndef GST_DISABLE_GST_DEBUG + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL, &entry) == ACAMERA_OK) { + guint8 level = entry.data.u8[0]; + gchar *level_str = NULL; + switch (level) { + case ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED: + level_str = g_strdup ("limited"); + break; + case ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL_FULL: + level_str = g_strdup ("full"); + break; + case ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY: + level_str = g_strdup ("legacy"); + break; + case ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL_3: + level_str = g_strdup ("3"); + break; + default: + level_str = g_strdup ("unknown"); + GST_WARNING_OBJECT (self, "Unknown supported hardware level (%d)", + level); + break; + } + + GST_DEBUG_OBJECT (self, "Hardware supported level: %s", level_str); + g_free (level_str); + } + + GST_DEBUG_OBJECT (self, "List of capabilities:"); + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_REQUEST_AVAILABLE_CAPABILITIES, &entry) == ACAMERA_OK) { + int i = 0; + for (; i < entry.count; i++) { + guint8 cap = entry.data.u8[i]; + gchar *cap_str = NULL; + switch (cap) { + case ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_BACKWARD_COMPATIBLE: + cap_str = g_strdup (" backward compatible"); + break; + case ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_MANUAL_SENSOR: + cap_str = g_strdup (" manual sensor"); + break; + case ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_MANUAL_POST_PROCESSING: + cap_str = g_strdup (" manual post processing"); + break; + case ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_RAW: + cap_str = g_strdup (" raw"); + break; + case ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_READ_SENSOR_SETTINGS: + cap_str = g_strdup (" read sensor settings"); + break; + case ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_BURST_CAPTURE: + cap_str = g_strdup (" burst capture"); + break; + case ACAMERA_REQUEST_AVAILABLE_CAPABILITIES_DEPTH_OUTPUT: + cap_str = g_strdup (" depth output"); + break; + default: + cap_str = g_strdup (" unknown"); + break; + } + GST_DEBUG_OBJECT (self, " %s(%u)", cap_str, cap); + g_free (cap_str); + } + } +#endif + + GST_DEBUG_OBJECT (self, "Checking available FPS ranges:"); + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES, + &fps_entry) == ACAMERA_OK) { + int i = 0; + for (; i < fps_entry.count; i += 2) { + GST_DEBUG_OBJECT (self, " (min: %d, max: %d)", fps_entry.data.i32[i], + fps_entry.data.i32[i + 1]); + } + } + + /* Only NV12 is acceptable. */ + format = gst_structure_new ("video/x-raw", + "format", G_TYPE_STRING, "NV12", NULL); + + GST_DEBUG_OBJECT (self, "Supported available stream configurations:"); + if (ACameraMetadata_getConstEntry (metadata, + ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS, + &entry) == ACAMERA_OK) { + gint i = 0; + for (; i < entry.count; i += 4) { + gint input = entry.data.i32[i + 3]; + gint fmt = entry.data.i32[i + 0]; + + if (input) + continue; + + if (fmt == AIMAGE_FORMAT_YUV_420_888) { + gint fps_idx = 0; + gint width = entry.data.i32[i + 1]; + gint height = entry.data.i32[i + 2]; + GstStructure *size = gst_structure_copy (format); + + GST_DEBUG_OBJECT (self, " (w: %d, h: %d)", width, height); + + gst_structure_set (size, + "width", G_TYPE_INT, width, + "height", G_TYPE_INT, height, + "interlaced", G_TYPE_BOOLEAN, FALSE, + "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, NULL); + + for (fps_idx = 0; fps_idx < fps_entry.count; fps_idx += 2) { + GstStructure *s = gst_structure_copy (size); + + if (fps_entry.data.i32[fps_idx] == fps_entry.data.i32[fps_idx + 1]) { + gst_structure_set (s, "framerate", GST_TYPE_FRACTION, + fps_entry.data.i32[fps_idx], 1, NULL); + } else { + gst_structure_set (s, "framerate", GST_TYPE_FRACTION_RANGE, + fps_entry.data.i32[fps_idx], 1, fps_entry.data.i32[fps_idx + 1], + 1, NULL); + } + gst_caps_append_structure (caps, s); + } + gst_structure_free (size); + } + } + } else { + GST_WARNING_OBJECT (self, " No available stream configurations detected"); + } + + gst_structure_free (format); + ACameraMetadata_free (metadata); + + GST_DEBUG_OBJECT (self, "%" GST_PTR_FORMAT, caps); + + GST_OBJECT_UNLOCK (self); + + return caps; +} + +static gboolean +gst_ahc2_src_unlock (GstBaseSrc * src) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + + GST_DEBUG_OBJECT (self, "flushing the queue"); + GST_OBJECT_LOCK (self); + gst_data_queue_set_flushing (self->outbound_queue, TRUE); + gst_data_queue_flush (self->outbound_queue); + GST_OBJECT_UNLOCK (self); + + return TRUE; +} + +static gboolean +gst_ahc2_src_unlock_stop (GstBaseSrc * src) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + + GST_DEBUG_OBJECT (self, "stop flushing the queue"); + GST_OBJECT_LOCK (self); + gst_data_queue_set_flushing (self->outbound_queue, FALSE); + gst_data_queue_flush (self->outbound_queue); + GST_OBJECT_UNLOCK (self); + + return TRUE; +} + +static GstFlowReturn +gst_ahc2_src_create (GstPushSrc * src, GstBuffer ** buffer) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + GstDataQueueItem *item; + + if (!gst_data_queue_pop (self->outbound_queue, &item)) { + GST_DEBUG_OBJECT (self, "We're flushing, item: %p", item); + return GST_FLOW_FLUSHING; + } + + *buffer = GST_BUFFER (item->object); + GST_TRACE_OBJECT (self, "Create function for buffer: %p", *buffer); + g_free (item); + + return GST_FLOW_OK; +} + +static void +gst_ahc2_src_camera_close (GstAHC2Src * self) +{ + GST_DEBUG_OBJECT (self, "Closing Camera"); + + if (self->image_reader) { + AImageReader_setImageListener (self->image_reader, NULL); + g_clear_pointer (&self->image_reader, (GDestroyNotify) AImageReader_delete); + } + + g_clear_pointer (&self->capture_sout, + (GDestroyNotify) ACaptureSessionOutput_free); + + g_clear_pointer (&self->camera_output_target, + (GDestroyNotify) ACameraOutputTarget_free); + + g_clear_pointer (&self->capture_request, + (GDestroyNotify) ACaptureRequest_free); + + g_clear_pointer (&self->capture_sout_container, + (GDestroyNotify) ACaptureSessionOutputContainer_free); + + g_clear_pointer (&self->camera_device, (GDestroyNotify) ACameraDevice_close); +} + +static gboolean +gst_ahc2_src_camera_open (GstAHC2Src * self) +{ + gboolean ret = FALSE; + camera_status_t status; + const gchar *camera_id; + + g_return_val_if_fail (self->camera_id_list != NULL, FALSE); + g_return_val_if_fail (self->camera_index < self->camera_id_list->numCameras, + FALSE); + + camera_id = self->camera_id_list->cameraIds[self->camera_index]; + GST_DEBUG_OBJECT (self, "Trying to open camera device (id: %s)", camera_id); + + status = ACameraManager_openCamera (self->camera_manager, camera_id, + &self->device_state_cb, &self->camera_device); + if (status != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to open camera device (id: %s, status: %s)", + camera_id, camera_status_t_to_string (status)); + goto out; + } + + status = ACameraDevice_createCaptureRequest (self->camera_device, + self->camera_template_type, &self->capture_request); + if (status != ACAMERA_OK) { + GST_ERROR_OBJECT (self, + "Failed to create camera request (camera id: %s, status: %s)", + camera_id, camera_status_t_to_string (status)); + goto out; + } + + status = + ACaptureSessionOutputContainer_create (&self->capture_sout_container); + if (status != ACAMERA_OK) { + GST_ERROR_OBJECT (self, + "Failed to create capture session output container (camera id: %s, status: %s)", + camera_id, camera_status_t_to_string (status)); + goto out; + } + + ret = TRUE; + +out: + if (!ret) + gst_ahc2_src_camera_close (self); + + return ret; +} + +static gboolean +gst_ahc2_src_start (GstBaseSrc * src) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + GST_OBJECT_LOCK (self); + + if (!gst_ahc2_src_camera_open (self)) { + goto out; + } + + self->previous_ts = GST_CLOCK_TIME_NONE; + self->started = TRUE; + +out: + GST_OBJECT_UNLOCK (self); + + return self->started; +} + +static gboolean +gst_ahc2_src_stop (GstBaseSrc * src) +{ + GstAHC2Src *self = GST_AHC2_SRC (src); + + GST_OBJECT_LOCK (self); + ACameraCaptureSession_abortCaptures (self->camera_capture_session); + gst_ahc2_src_camera_close (self); + + gst_data_queue_flush (self->outbound_queue); + + self->started = FALSE; + GST_OBJECT_UNLOCK (self); + + return TRUE; +} + +static void +gst_ahc2_src_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstAHC2Src *self = GST_AHC2_SRC (object); + + switch ((GstAHC2SrcProperty) prop_id) { + case PROP_CAMERA_INDEX: + self->camera_index = g_value_get_int (value); + break; + case PROP_CAMERA_TEMPLATE_TYPE: + self->camera_template_type = g_value_get_int (value); + break; + case PROP_MAX_IMAGES: + self->max_images = g_value_get_int (value); + break; + case PROP_COLOR_TONE_MODE:{ + GstPhotographyColorToneMode tone = g_value_get_enum (value); + gst_ahc2_src_set_color_tone_mode (GST_PHOTOGRAPHY (self), tone); + break; + } + case PROP_EV_COMPENSATION:{ + gfloat ev = g_value_get_float (value); + gst_ahc2_src_set_ev_compensation (GST_PHOTOGRAPHY (self), ev); + break; + } + case PROP_SCENE_MODE:{ + GstPhotographySceneMode scene = g_value_get_enum (value); + gst_ahc2_src_set_scene_mode (GST_PHOTOGRAPHY (self), scene); + break; + } + case PROP_WHITE_BALANCE_MODE:{ + GstPhotographyWhiteBalanceMode wb = g_value_get_enum (value); + gst_ahc2_src_set_white_balance_mode (GST_PHOTOGRAPHY (self), wb); + break; + } + case PROP_FLASH_MODE:{ + GstPhotographyFlashMode flash = g_value_get_enum (value); + gst_ahc2_src_set_flash_mode (GST_PHOTOGRAPHY (self), flash); + break; + } + case PROP_FLICKER_MODE:{ + GstPhotographyFlickerReductionMode flicker = g_value_get_enum (value); + gst_ahc2_src_set_flicker_mode (GST_PHOTOGRAPHY (self), flicker); + break; + } + case PROP_FOCUS_MODE:{ + GstPhotographyFocusMode focus = g_value_get_enum (value); + gst_ahc2_src_set_focus_mode (GST_PHOTOGRAPHY (self), focus); + break; + } + case PROP_ZOOM:{ + gfloat zoom = g_value_get_float (value); + gst_ahc2_src_set_zoom (GST_PHOTOGRAPHY (self), zoom); + break; + } + case PROP_EXPOSURE_MODE:{ + GstPhotographyExposureMode exposure_mode = g_value_get_enum (value); + gst_ahc2_src_set_exposure_mode (GST_PHOTOGRAPHY (self), exposure_mode); + break; + } + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_ahc2_src_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstAHC2Src *self = GST_AHC2_SRC (object); + + switch ((GstAHC2SrcProperty) prop_id) { + case PROP_CAMERA_INDEX: + g_value_set_int (value, self->camera_index); + break; + case PROP_N_CAMERAS: + if (self->camera_id_list != NULL) { + g_value_set_int (value, self->camera_id_list->numCameras); + } else { + GST_WARNING_OBJECT (self, "Camera list doesn't exist."); + g_value_set_int (value, 0); + } + break; + case PROP_CAMERA_TEMPLATE_TYPE: + g_value_set_int (value, self->camera_template_type); + break; + case PROP_MAX_IMAGES: + g_value_set_int (value, self->max_images); + break; + case PROP_CAPABILITIES:{ + GstPhotographyCaps caps; + + caps = gst_ahc2_src_get_capabilities (GST_PHOTOGRAPHY (self)); + g_value_set_ulong (value, caps); + + break; + } + case PROP_COLOR_TONE_MODE:{ + GstPhotographyColorToneMode tone; + if (gst_ahc2_src_get_color_tone_mode (GST_PHOTOGRAPHY (self), &tone)) + g_value_set_enum (value, tone); + break; + } + case PROP_EV_COMPENSATION:{ + gfloat ev; + + if (gst_ahc2_src_get_ev_compensation (GST_PHOTOGRAPHY (self), &ev)) + g_value_set_float (value, ev); + + break; + } + case PROP_WHITE_BALANCE_MODE:{ + GstPhotographyWhiteBalanceMode wb; + if (gst_ahc2_src_get_white_balance_mode (GST_PHOTOGRAPHY (self), &wb)) + g_value_set_enum (value, wb); + break; + } + case PROP_FLASH_MODE:{ + GstPhotographyFlashMode flash; + if (gst_ahc2_src_get_flash_mode (GST_PHOTOGRAPHY (self), &flash)) + g_value_set_enum (value, flash); + break; + } + case PROP_FLICKER_MODE:{ + GstPhotographyFlickerReductionMode flicker; + if (gst_ahc2_src_get_flicker_mode (GST_PHOTOGRAPHY (self), &flicker)) + g_value_set_enum (value, flicker); + break; + } + case PROP_FOCUS_MODE:{ + GstPhotographyFocusMode focus; + if (gst_ahc2_src_get_focus_mode (GST_PHOTOGRAPHY (self), &focus)) + g_value_set_enum (value, focus); + break; + } + case PROP_SCENE_MODE:{ + GstPhotographySceneMode scene; + if (gst_ahc2_src_get_scene_mode (GST_PHOTOGRAPHY (self), &scene)) + g_value_set_enum (value, scene); + break; + } + case PROP_ZOOM:{ + gfloat zoom; + if (gst_ahc2_src_get_zoom (GST_PHOTOGRAPHY (self), &zoom)) + g_value_set_float (value, zoom); + break; + } + case PROP_EXPOSURE_MODE:{ + GstPhotographyExposureMode exposure_mode; + if (gst_ahc2_src_get_exposure_mode (GST_PHOTOGRAPHY (self), + &exposure_mode)) + g_value_set_enum (value, exposure_mode); + break; + } + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_ahc2_src_finalize (GObject * object) +{ + GstAHC2Src *self = GST_AHC2_SRC (object); + + g_clear_pointer (&self->camera_id_list, + (GDestroyNotify) ACameraManager_deleteCameraIdList); + + g_clear_pointer (&self->camera_manager, + (GDestroyNotify) ACameraManager_delete); + + g_clear_pointer (&self->outbound_queue, gst_object_unref); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +data_queue_item_free (GstDataQueueItem * item) +{ + gst_mini_object_unref (item->object); + g_free (item); +} + +#if 0 +ACameraMetadata *metadata = NULL; +if (ACameraManager_getCameraCharacteristics (self->camera_manager, + self->camera_id_list->cameraIds[self->camera_index], + &metadata) != ACAMERA_OK) { + return; +} +ACameraMetadata_const_entry orientation; +ACameraMetadata_getConstEntry (metadata, ACAMERA_SENSOR_ORIENTATION, + &orientation); +int32_t angle = orientation.data.i32[0]; +GST_DEBUG ("Our angle is: %d", angle); +#endif + +static void +_aimage_to_gstbuffer (GstAHC2Src * self, AImage * image, GstBuffer * buffer) +{ + GstWrappedAImage *wrapped_aimage; + + guint8 *y_data; + guint8 *u_data; + guint8 *v_data; + + gint y_length; + gint u_length; + gint v_length; + + gint y_stride; + gint u_stride; + gint v_stride; + + gint u_pixelstride; + gint v_pixelstride; + + GstMemory *y_mem; + GstMemory *uv_mem; + + gint n_planes; + + AImage_getNumberOfPlanes (image, &n_planes); + if (n_planes != 3) { + GST_ERROR_OBJECT (self, "Not getting 3 planes!!"); + g_assert_not_reached (); + } + + AImage_getPlaneData (image, 0, &y_data, &y_length); + AImage_getPlaneData (image, 1, &u_data, &u_length); + AImage_getPlaneData (image, 2, &v_data, &v_length); + + AImage_getPlaneRowStride (image, 0, &y_stride); + AImage_getPlaneRowStride (image, 1, &u_stride); + AImage_getPlaneRowStride (image, 2, &v_stride); + + AImage_getPlanePixelStride (image, 1, &u_pixelstride); + AImage_getPlanePixelStride (image, 2, &v_pixelstride); + + wrapped_aimage = g_new0 (GstWrappedAImage, 1); + wrapped_aimage->refcount = 1; + wrapped_aimage->ahc2src = g_object_ref (self); + wrapped_aimage->image = image; + + /* append the Y memory */ + y_mem = gst_memory_new_wrapped (GST_MEMORY_FLAG_READONLY, + y_data, y_length, 0, y_length, + gst_wrapped_aimage_ref (wrapped_aimage), + (GDestroyNotify) gst_wrapped_aimage_unref); + gst_buffer_append_memory (buffer, y_mem); + + /* if all of this is true, we can safly assume a NV12 layout, where the + whole memory for the UV plane is contained inside both u_data and v_data, + with only 1 pixel moved */ + if (u_length == v_length && /* we have the same length */ + u_stride == v_stride && /* we have the same stride */ + ((u_data + 1 == v_data) || (v_data + 1 == u_data)) && /* v_data is basically pointing one pixel into u_data (or v.v), otherwise identical, expected for NV12/NV21 */ + u_pixelstride == 2 && v_pixelstride == 2) { /* the packing you expect from inteleaved U/V/U/V in NV12 */ + + gsize offset[GST_VIDEO_MAX_PLANES] = { 0, y_length, 0, 0 }; + gint stride[GST_VIDEO_MAX_PLANES] = { y_stride, u_stride, 0, 0 }; + GstVideoAlignment align; + gst_video_alignment_reset (&align); + align.padding_right = y_stride - self->width; + + uv_mem = gst_memory_new_wrapped (GST_MEMORY_FLAG_READONLY, + u_data, y_length / 2, 0, y_length / 2, + gst_wrapped_aimage_ref (wrapped_aimage), + (GDestroyNotify) gst_wrapped_aimage_unref); + gst_buffer_append_memory (buffer, uv_mem); + + gst_video_meta_set_alignment (gst_buffer_add_video_meta_full (buffer, + GST_VIDEO_FRAME_FLAG_NONE, GST_VIDEO_FORMAT_NV12, self->width, + self->height, 2, offset, stride), align); + + GST_INFO_OBJECT (self, + "y_length=%d u_length=%d, v_length=%d, " + "y_stride=%d, u_stride=%d, v_stride=%d, " + "y_data=%p, u_data=%p, v_data=%p, " + "width=%d, height=%d, " + "u_pixelstride=%d, v_pixelstride=%d", + y_length, u_length, v_length, y_stride, u_stride, v_stride, + y_data, u_data, v_data, self->width, self->height, + u_pixelstride, v_pixelstride); + } else { + /* we need this assumption to be true for now! */ + GST_ERROR_OBJECT (self, "Not getting the NV12 memory-layout we expected: " + "y_length=%d u_length=%d, v_length=%d, " + "y_stride=%d, u_stride=%d, v_stride=%d, " + "y_data=%p, u_data=%p, v_data=%p, " + "width=%d, height=%d, " + "u_pixelstride=%d, v_pixelstride=%d", + y_length, u_length, v_length, y_stride, u_stride, v_stride, + y_data, u_data, v_data, self->width, self->height, + u_pixelstride, v_pixelstride); + g_assert_not_reached (); + } + + gst_wrapped_aimage_unref (wrapped_aimage); +} + +static void +image_reader_on_image_available (void *context, AImageReader * reader) +{ + GstAHC2Src *self = GST_AHC2_SRC (context); + + GstBuffer *buffer; + GstDataQueueItem *item; + GstClockTime duration = GST_CLOCK_TIME_NONE; + GstClockTime current_ts = GST_CLOCK_TIME_NONE; + GstClock *clock; + AImage *image = NULL; + + if ((clock = GST_ELEMENT_CLOCK (self))) { + GstClockTime base_time = GST_ELEMENT_CAST (self)->base_time; + gst_object_ref (clock); + current_ts = gst_clock_get_time (clock) - base_time; + gst_object_unref (clock); + } + + GST_OBJECT_LOCK (self); + + if (AImageReader_acquireLatestImage (reader, &image) != AMEDIA_OK) { + GST_DEBUG_OBJECT (self, "No image available"); + GST_OBJECT_UNLOCK (self); + return; + } + + GST_DEBUG_OBJECT (self, "Acquired an image (ts: %" GST_TIME_FORMAT ")", + GST_TIME_ARGS (current_ts)); + +#ifndef GST_DISABLE_GST_DEBUG + { + gint64 image_ts; + /* AImageReader has its own clock. */ + AImage_getTimestamp (image, &image_ts); + GST_DEBUG_OBJECT (self, " AImage internal ts: %" GST_TIME_FORMAT ")", + GST_TIME_ARGS (image_ts)); + } +#endif + + if (!self->started || GST_CLOCK_TIME_IS_VALID (self->previous_ts)) { + duration = current_ts - self->previous_ts; + self->previous_ts = current_ts; + } else { + self->previous_ts = current_ts; + /* Dropping first image to calculate duration */ + AImage_delete (image); + GST_DEBUG_OBJECT (self, "Droping image (reason: %s)", + self->started ? "first frame" : "not yet started"); + GST_OBJECT_UNLOCK (self); + return; + } + + buffer = gst_buffer_new (); + GST_BUFFER_DURATION (buffer) = duration; + GST_BUFFER_PTS (buffer) = current_ts; + + _aimage_to_gstbuffer (self, image, buffer); + + g_atomic_int_add (&self->referenced_images, 1); + GST_TRACE_OBJECT (self, "Created image:%p, referenced_images: %d", image, + self->referenced_images); + + item = g_new0 (GstDataQueueItem, 1); + item->object = GST_MINI_OBJECT (buffer); + item->size = gst_buffer_get_size (buffer); + item->visible = TRUE; + item->destroy = (GDestroyNotify) data_queue_item_free; + + GST_TRACE_OBJECT (self, "About to push buffer: %p on queue", buffer); + if (!gst_data_queue_push (self->outbound_queue, item)) { + item->destroy (item); + GST_DEBUG_OBJECT (self, "Failed to push item because we're flushing"); + } + + GST_OBJECT_UNLOCK (self); + + GST_TRACE_OBJECT (self, + "created buffer %p from image callback %" G_GSIZE_FORMAT ", ts %" + GST_TIME_FORMAT ", dur %" GST_TIME_FORMAT + ", offset %" G_GINT64_FORMAT ", offset_end %" G_GINT64_FORMAT, + buffer, gst_buffer_get_size (buffer), + GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buffer)), + GST_TIME_ARGS (GST_BUFFER_DURATION (buffer)), + GST_BUFFER_OFFSET (buffer), GST_BUFFER_OFFSET_END (buffer)); +} + +static void +camera_device_on_disconnected (void *context, ACameraDevice * device) +{ + GstAHC2Src *self = GST_AHC2_SRC (context); + + GST_DEBUG_OBJECT (self, "Camera[%s] is disconnected", + ACameraDevice_getId (device)); +} + +static void +camera_device_on_error (void *context, ACameraDevice * device, int error) +{ + GstAHC2Src *self = GST_AHC2_SRC (context); + + GST_ERROR_OBJECT (self, "Error (err: %s, code: %d) on Camera[%s]", + camera_error_to_string (error), error, ACameraDevice_getId (device)); +} + +static void +capture_session_on_ready (void *context, ACameraCaptureSession * session) +{ + GstAHC2Src *self = GST_AHC2_SRC (context); + + GST_DEBUG_OBJECT (self, "Camera[%s] capture session is ready", + ACameraDevice_getId (self->camera_device)); +} + +static void +capture_session_on_active (void *context, ACameraCaptureSession * session) +{ + GstAHC2Src *self = GST_AHC2_SRC (context); + + GST_DEBUG_OBJECT (self, "Camera[%s] capture session is activated", + ACameraDevice_getId (self->camera_device)); +} + +static const gchar * +gst_ahc2_src_get_camera_id_by_index (GstAHC2Src * self, gint idx) +{ + g_return_val_if_fail (self->camera_id_list != NULL, NULL); + g_return_val_if_fail (idx >= self->camera_id_list->numCameras, NULL); + + return self->camera_id_list->cameraIds[idx]; +} + +static void +gst_ahc2_src_class_init (GstAHC2SrcClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass); + GstBaseSrcClass *gstbasesrc_class = GST_BASE_SRC_CLASS (klass); + GstPushSrcClass *gstpushsrc_class = GST_PUSH_SRC_CLASS (klass); + + gobject_class->set_property = gst_ahc2_src_set_property; + gobject_class->get_property = gst_ahc2_src_get_property; + gobject_class->finalize = gst_ahc2_src_finalize; + + /** + * GstAHC2Src:camera-index: + * + * The Camera index to be connected. + */ + properties[PROP_CAMERA_INDEX] = + g_param_spec_int ("camera-index", "Camera Index", + "The camera device index to open", 0, G_MAXINT, 0, + G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY | G_PARAM_STATIC_STRINGS); + + /** + * GstAHC2Src:n-cameras: + * + * The number of connected camera devices. + */ + properties[PROP_N_CAMERAS] = + g_param_spec_int ("n-cameras", "Number of Cameras", + "The number of connected camera devices", 0, G_MAXINT, 0, + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS); + + /** + * GstAHC2Src:camera-template-type: + * The suitable template type for the purpose of running camera. + */ + properties[PROP_CAMERA_TEMPLATE_TYPE] = + g_param_spec_int ("camera-template-type", "Camera Template Type", + "A request template for the camera running purpose", -1, G_MAXINT, + TEMPLATE_PREVIEW, + G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY | G_PARAM_STATIC_STRINGS); + + /** + * GstAHC2Src:max-images: + * The number of buffering images. + */ + properties[PROP_MAX_IMAGES] = + g_param_spec_int ("max-images", "Max Images", + "The number of buffering images", 0, G_MAXINT, + DEFAULT_MAX_IMAGES, + G_PARAM_READWRITE | GST_PARAM_MUTABLE_PAUSED | G_PARAM_STATIC_STRINGS); + + g_object_class_install_properties (gobject_class, PROP_N_INSTALL, properties); + + /* Override GstPhotography properties */ + g_object_class_override_property (gobject_class, PROP_ANALOG_GAIN, + GST_PHOTOGRAPHY_PROP_ANALOG_GAIN); + properties[PROP_ANALOG_GAIN] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_ANALOG_GAIN); + + g_object_class_override_property (gobject_class, PROP_APERTURE, + GST_PHOTOGRAPHY_PROP_APERTURE); + properties[PROP_APERTURE] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_APERTURE); + + g_object_class_override_property (gobject_class, PROP_CAPABILITIES, + GST_PHOTOGRAPHY_PROP_CAPABILITIES); + properties[PROP_CAPABILITIES] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_CAPABILITIES); + + g_object_class_override_property (gobject_class, PROP_COLOR_TEMPERATURE, + GST_PHOTOGRAPHY_PROP_COLOR_TEMPERATURE); + properties[PROP_COLOR_TEMPERATURE] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_COLOR_TEMPERATURE); + + g_object_class_override_property (gobject_class, PROP_COLOR_TONE_MODE, + GST_PHOTOGRAPHY_PROP_COLOR_TONE); + properties[PROP_COLOR_TONE_MODE] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_COLOR_TONE); + + g_object_class_override_property (gobject_class, PROP_EV_COMPENSATION, + GST_PHOTOGRAPHY_PROP_EV_COMP); + properties[PROP_EV_COMPENSATION] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_EV_COMP); + + g_object_class_override_property (gobject_class, PROP_EXPOSURE_TIME, + GST_PHOTOGRAPHY_PROP_EXPOSURE_TIME); + properties[PROP_EXPOSURE_TIME] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_EXPOSURE_TIME); + + g_object_class_override_property (gobject_class, PROP_FLASH_MODE, + GST_PHOTOGRAPHY_PROP_FLASH_MODE); + properties[PROP_FLASH_MODE] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_FLASH_MODE); + + g_object_class_override_property (gobject_class, PROP_FLICKER_MODE, + GST_PHOTOGRAPHY_PROP_FLICKER_MODE); + properties[PROP_FLICKER_MODE] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_FLICKER_MODE); + + g_object_class_override_property (gobject_class, PROP_FOCUS_MODE, + GST_PHOTOGRAPHY_PROP_FOCUS_MODE); + properties[PROP_FOCUS_MODE] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_FOCUS_MODE); + + g_object_class_override_property (gobject_class, + PROP_IMAGE_CAPTURE_SUPPORTED_CAPS, + GST_PHOTOGRAPHY_PROP_IMAGE_CAPTURE_SUPPORTED_CAPS); + properties[PROP_IMAGE_CAPTURE_SUPPORTED_CAPS] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_IMAGE_CAPTURE_SUPPORTED_CAPS); + + g_object_class_override_property (gobject_class, + PROP_IMAGE_PREVIEW_SUPPORTED_CAPS, + GST_PHOTOGRAPHY_PROP_IMAGE_PREVIEW_SUPPORTED_CAPS); + properties[PROP_IMAGE_PREVIEW_SUPPORTED_CAPS] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_IMAGE_PREVIEW_SUPPORTED_CAPS); + + g_object_class_override_property (gobject_class, PROP_ISO_SPEED, + GST_PHOTOGRAPHY_PROP_ISO_SPEED); + properties[PROP_ISO_SPEED] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_ISO_SPEED); + + g_object_class_override_property (gobject_class, PROP_LENS_FOCUS, + GST_PHOTOGRAPHY_PROP_LENS_FOCUS); + properties[PROP_LENS_FOCUS] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_LENS_FOCUS); + + g_object_class_override_property (gobject_class, PROP_MAX_EXPOSURE_TIME, + GST_PHOTOGRAPHY_PROP_MAX_EXPOSURE_TIME); + properties[PROP_MAX_EXPOSURE_TIME] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_MAX_EXPOSURE_TIME); + + g_object_class_override_property (gobject_class, PROP_MIN_EXPOSURE_TIME, + GST_PHOTOGRAPHY_PROP_MIN_EXPOSURE_TIME); + properties[PROP_MIN_EXPOSURE_TIME] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_MIN_EXPOSURE_TIME); + + g_object_class_override_property (gobject_class, PROP_NOISE_REDUCTION, + GST_PHOTOGRAPHY_PROP_NOISE_REDUCTION); + properties[PROP_NOISE_REDUCTION] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_NOISE_REDUCTION); + + g_object_class_override_property (gobject_class, PROP_SCENE_MODE, + GST_PHOTOGRAPHY_PROP_SCENE_MODE); + properties[PROP_SCENE_MODE] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_SCENE_MODE); + + g_object_class_override_property (gobject_class, PROP_WHITE_BALANCE_MODE, + GST_PHOTOGRAPHY_PROP_WB_MODE); + properties[PROP_WHITE_BALANCE_MODE] = + g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_WB_MODE); + + g_object_class_override_property (gobject_class, PROP_WHITE_POINT, + GST_PHOTOGRAPHY_PROP_WHITE_POINT); + properties[PROP_WHITE_POINT] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_WHITE_POINT); + + g_object_class_override_property (gobject_class, PROP_ZOOM, + GST_PHOTOGRAPHY_PROP_ZOOM); + properties[PROP_ZOOM] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_ZOOM); + + g_object_class_override_property (gobject_class, + PROP_EXPOSURE_MODE, GST_PHOTOGRAPHY_PROP_EXPOSURE_MODE); + properties[PROP_EXPOSURE_MODE] = g_object_class_find_property (gobject_class, + GST_PHOTOGRAPHY_PROP_EXPOSURE_MODE); + + signals[SIG_GET_CAMERA_ID_BY_INDEX] = + g_signal_new ("get-camera-id-by-index", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, G_STRUCT_OFFSET (GstAHC2SrcClass, + get_camera_id_by_index), NULL, NULL, NULL, + G_TYPE_CHAR, 1, G_TYPE_INT); + + gst_element_class_add_static_pad_template (gstelement_class, &src_template); + gst_element_class_set_static_metadata (gstelement_class, + "Android Camera2 Source", "Source/Video/Device", + "Source element for Android hardware camers 2 API", + "Justin Kim "); + + gstbasesrc_class->get_caps = GST_DEBUG_FUNCPTR (gst_ahc2_src_get_caps); + gstbasesrc_class->set_caps = GST_DEBUG_FUNCPTR (gst_ahc2_src_set_caps); + gstbasesrc_class->fixate = GST_DEBUG_FUNCPTR (gst_ahc2_src_fixate); + gstbasesrc_class->start = GST_DEBUG_FUNCPTR (gst_ahc2_src_start); + gstbasesrc_class->stop = GST_DEBUG_FUNCPTR (gst_ahc2_src_stop); + gstbasesrc_class->unlock = GST_DEBUG_FUNCPTR (gst_ahc2_src_unlock); + gstbasesrc_class->unlock_stop = GST_DEBUG_FUNCPTR (gst_ahc2_src_unlock_stop); + + gstpushsrc_class->create = GST_DEBUG_FUNCPTR (gst_ahc2_src_create); + + klass->get_camera_id_by_index = gst_ahc2_src_get_camera_id_by_index; + + GST_DEBUG_CATEGORY_INIT (gst_debug_ahc2_src, "ahc2src", 0, "AHC2 Source"); +} + +static gboolean +data_queue_check_full_cb (GstDataQueue * queue, guint visible, + guint bytes, guint64 time, gpointer checkdata) +{ + return FALSE; +} + +static void +gst_ahc2_src_init (GstAHC2Src * self) +{ + self->camera_template_type = TEMPLATE_PREVIEW; + self->max_images = DEFAULT_MAX_IMAGES; + self->outbound_queue = + gst_data_queue_new (data_queue_check_full_cb, NULL, NULL, NULL); + + gst_base_src_set_format (GST_BASE_SRC (self), GST_FORMAT_TIME); + gst_base_src_set_live (GST_BASE_SRC (self), TRUE); + + self->camera_manager = ACameraManager_create (); + + if (ACameraManager_getCameraIdList (self->camera_manager, + &self->camera_id_list) != ACAMERA_OK) { + GST_ERROR_OBJECT (self, "Failed to get camera id list"); + goto fail; + } + + if (self->camera_id_list->numCameras < 1) { + GST_WARNING_OBJECT (self, "No camera device detected."); + goto fail; + } + + self->device_state_cb.context = self; + self->device_state_cb.onDisconnected = camera_device_on_disconnected; + self->device_state_cb.onError = camera_device_on_error; + + self->session_state_cb.context = self; + self->session_state_cb.onReady = capture_session_on_ready; + self->session_state_cb.onActive = capture_session_on_active; + + self->image_reader_listener.context = self; + self->image_reader_listener.onImageAvailable = + image_reader_on_image_available; + +#ifndef GST_DISABLE_GST_DEBUG + { + int idx = 0; + + GST_DEBUG_OBJECT (self, "detected %d cameras", + self->camera_id_list->numCameras); + for (; idx < self->camera_id_list->numCameras; idx++) { + GST_DEBUG_OBJECT (self, " camera device[%d]: %s", idx, + self->camera_id_list->cameraIds[idx]); + } + } +#endif + + return; + +fail: + g_clear_pointer (&self->camera_id_list, + (GDestroyNotify) ACameraManager_deleteCameraIdList); + + g_clear_pointer (&self->camera_manager, + (GDestroyNotify) ACameraManager_delete); +} + +static void +gst_ahc2_src_photography_init (gpointer g_iface, gpointer iface_data) +{ + GstPhotographyInterface *iface = g_iface; + + iface->get_capabilities = gst_ahc2_src_get_capabilities; + + iface->get_color_tone_mode = gst_ahc2_src_get_color_tone_mode; + iface->set_color_tone_mode = gst_ahc2_src_set_color_tone_mode; + + iface->set_autofocus = gst_ahc2_src_set_autofocus; + iface->get_focus_mode = gst_ahc2_src_get_focus_mode; + iface->set_focus_mode = gst_ahc2_src_set_focus_mode; + + iface->get_exposure_mode = gst_ahc2_src_get_exposure_mode; + iface->set_exposure_mode = gst_ahc2_src_set_exposure_mode; + + iface->get_flash_mode = gst_ahc2_src_get_flash_mode; + iface->set_flash_mode = gst_ahc2_src_set_flash_mode; + + iface->get_flicker_mode = gst_ahc2_src_get_flicker_mode; + iface->set_flicker_mode = gst_ahc2_src_set_flicker_mode; + + iface->get_scene_mode = gst_ahc2_src_get_scene_mode; + iface->set_scene_mode = gst_ahc2_src_set_scene_mode; + + iface->get_white_balance_mode = gst_ahc2_src_get_white_balance_mode; + iface->set_white_balance_mode = gst_ahc2_src_set_white_balance_mode; + + iface->get_ev_compensation = gst_ahc2_src_get_ev_compensation; + iface->set_ev_compensation = gst_ahc2_src_set_ev_compensation; + + iface->get_zoom = gst_ahc2_src_get_zoom; + iface->set_zoom = gst_ahc2_src_set_zoom; +} diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.h b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.h new file mode 100644 index 00000000000..f222ffb98c4 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstahc2src.h @@ -0,0 +1,56 @@ +/* GStreamer android.hardware.Camera2 Source + * Copyright (C) 2017, Collabora Ltd. + * Author:Justin Kim + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_AHC2_SRC_H__ +#define __GST_AHC2_SRC_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_AHC2_SRC (gst_ahc2_src_get_type ()) +#define GST_IS_AHC2_SRC(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_AHC2_SRC)) +#define GST_IS_AHC2_SRC_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_AHC2_SRC)) +#define GST_AHC2_SRC_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_AHC2_SRC, GstAHC2SrcClass)) +#define GST_AHC2_SRC(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_AHC2_SRC, GstAHC2Src)) +#define GST_AHC2_SRC_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_AHC2_SRC, GstAHC2SrcClass)) +#define GST_AHC2_SRC_CAST(obj) ((GstAHC2Src*)(obj)) +#define GST_AHC2_SRC_CLASS_CAST(klass) ((GstAHC2SrcClass*)(klass)) + +typedef struct _GstAHC2Src GstAHC2Src; +typedef struct _GstAHC2SrcClass GstAHC2SrcClass; +typedef struct _GstAHC2SrcPrivate GstAHC2SrcPrivate; + +struct _GstAHC2SrcClass { + GstPushSrcClass parent_class; + + /* actions */ + const gchar* (*get_camera_id_by_index) (GstAHC2Src *self, gint idx); + + gpointer _gst_reserved[GST_PADDING_LARGE]; +}; + +GST_EXPORT +GType gst_ahc2_src_get_type (void); + +G_END_DECLS + +#endif /* __GST_AHC2_SRC_H__ */ diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstahcsrc.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstahcsrc.c index 448e11827d2..67349b6a821 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstahcsrc.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstahcsrc.c @@ -447,12 +447,10 @@ gst_ahc_src_class_init (GstAHCSrcClass * klass) properties[PROP_APERTURE] = g_object_class_find_property (gobject_class, GST_PHOTOGRAPHY_PROP_APERTURE); -#if 0 g_object_class_override_property (gobject_class, PROP_EXPOSURE_MODE, GST_PHOTOGRAPHY_PROP_EXPOSURE_MODE); - properties[PROP_EXPOSURE] = g_object_class_find_property (gobject_class, + properties[PROP_EXPOSURE_MODE] = g_object_class_find_property (gobject_class, GST_PHOTOGRAPHY_PROP_EXPOSURE_MODE); -#endif g_object_class_override_property (gobject_class, PROP_IMAGE_CAPTURE_SUPPORTED_CAPS, diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamc-constants.h b/subprojects/gst-plugins-bad/sys/androidmedia/gstamc-constants.h index 95a965aed71..f7aaa6aa61d 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamc-constants.h +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamc-constants.h @@ -29,7 +29,7 @@ enum { - BUFFER_FLAG_SYNC_FRAME = 1, + BUFFER_FLAG_KEY_FRAME = 1, BUFFER_FLAG_CODEC_CONFIG = 2, BUFFER_FLAG_END_OF_STREAM = 4, BUFFER_FLAG_PARTIAL_FRAME = 8, diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c index 2da4657bd54..196ea4ded7c 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamc.c @@ -32,9 +32,13 @@ #ifdef HAVE_JNI_H #include "gstahcsrc.h" #include "gstahssrc.h" +#include "gstamdeviceprovider.h" #include "gstjniutils.h" #endif +#include "gstahc2src.h" +#include "gstacamdeviceprovider.h" + #include "gstamc.h" #include "gstamc-constants.h" @@ -306,7 +310,7 @@ scan_codecs (GstPlugin * plugin) if (!accepted_color_formats (gst_codec_type, is_encoder)) { if (!ignore_unknown_color_formats) { gst_codec_info->gl_output_only = TRUE; - GST_WARNING + GST_DEBUG ("%s %s has unknown color formats, only direct rendering will be supported", gst_codec_type->mime, is_encoder ? "encoder" : "decoder"); } @@ -549,7 +553,7 @@ accepted_color_formats (GstAmcCodecType * type, gboolean is_encoder) } if (!found) { - GST_ERROR ("Unknown color format 0x%x, ignoring", type->color_formats[i]); + GST_DEBUG ("Unknown color format 0x%x, ignoring", type->color_formats[i]); } } @@ -1932,6 +1936,27 @@ plugin_init (GstPlugin * plugin) init_ok = TRUE; #endif + if (gst_element_register (plugin, "ahc2src", GST_RANK_NONE, + GST_TYPE_AHC2_SRC)) { + init_ok = TRUE; + } else { + GST_ERROR ("Failed to register android camera2 source"); + } + + if (GST_DEVICE_PROVIDER_REGISTER (acamdeviceprovider, plugin)) { + init_ok = TRUE; + } else { + GST_ERROR ("Failed to register android camera device provider"); + } + +#ifdef HAVE_JNI_H + if (GST_DEVICE_PROVIDER_REGISTER (amdeviceprovider, plugin)) { + init_ok = TRUE; + } else { + GST_ERROR ("Failed to register android AudioManager device provider"); + } +#endif + return init_ok; } @@ -2019,7 +2044,7 @@ gst_amc_codec_info_to_caps (const GstAmcCodecInfo * codec_info, gst_amc_aac_profile_to_string (type->profile_levels[j].profile); if (!profile) { - GST_ERROR ("Unable to map AAC profile 0x%08x", + GST_DEBUG ("Unable to map AAC profile 0x%08x", type->profile_levels[j].profile); continue; } @@ -2071,7 +2096,7 @@ gst_amc_codec_info_to_caps (const GstAmcCodecInfo * codec_info, "parsed", G_TYPE_BOOLEAN, TRUE, NULL); encoded_ret = gst_caps_merge_structure (encoded_ret, tmp); } else { - GST_WARNING ("Unsupported mimetype '%s'", type->mime); + GST_DEBUG ("Unsupported mimetype '%s'", type->mime); } } } else if (g_str_has_prefix (type->mime, "video/")) { @@ -2090,7 +2115,7 @@ gst_amc_codec_info_to_caps (const GstAmcCodecInfo * codec_info, gst_amc_color_format_to_video_format (codec_info, type->mime, type->color_formats[j]); if (format == GST_VIDEO_FORMAT_UNKNOWN) { - GST_WARNING ("Unknown color format 0x%08x for codec %s", + GST_DEBUG ("Unknown color format 0x%08x for codec %s", type->color_formats[j], type->mime); continue; } @@ -2267,7 +2292,7 @@ gst_amc_codec_info_to_caps (const GstAmcCodecInfo * codec_info, profile, &alternative); if (!profile) { - GST_ERROR ("Unable to map H264 profile 0x%08x", + GST_DEBUG ("Unable to map H264 profile 0x%08x", type->profile_levels[j].profile); continue; } @@ -2343,7 +2368,7 @@ gst_amc_codec_info_to_caps (const GstAmcCodecInfo * codec_info, profile); if (!profile) { - GST_ERROR ("Unable to map H265 profile 0x%08x", + GST_DEBUG ("Unable to map H265 profile 0x%08x", type->profile_levels[j].profile); continue; } @@ -2424,7 +2449,7 @@ gst_amc_codec_info_to_caps (const GstAmcCodecInfo * codec_info, encoded_ret = gst_caps_merge_structure (encoded_ret, tmp); } else { - GST_WARNING ("Unsupported mimetype '%s'", type->mime); + GST_DEBUG ("Unsupported mimetype '%s'", type->mime); } } } diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcaudiodec.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcaudiodec.c index 398a6e1bd3b..4a88e3fc59f 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcaudiodec.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcaudiodec.c @@ -1179,7 +1179,7 @@ gst_amc_audio_dec_handle_frame (GstAudioDecoder * decoder, GstBuffer * inbuf) if (offset == 0) { if (!GST_BUFFER_FLAG_IS_SET (inbuf, GST_BUFFER_FLAG_DELTA_UNIT)) - buffer_info.flags |= BUFFER_FLAG_SYNC_FRAME; + buffer_info.flags |= BUFFER_FLAG_KEY_FRAME; } offset += buffer_info.size; diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideodec.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideodec.c index e9230fb3868..00b153a8089 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideodec.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideodec.c @@ -2212,7 +2212,7 @@ gst_amc_video_dec_handle_frame (GstVideoDecoder * decoder, BufferIdentification *id = buffer_identification_new (timestamp + timestamp_offset); if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame)) - buffer_info.flags |= BUFFER_FLAG_SYNC_FRAME; + buffer_info.flags |= BUFFER_FLAG_KEY_FRAME; gst_video_codec_frame_set_user_data (frame, id, (GDestroyNotify) buffer_identification_free); } diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c index d04a1d2684c..5d7f9e68a11 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamcvideoenc.c @@ -94,7 +94,7 @@ static GstFlowReturn gst_amc_video_enc_finish (GstVideoEncoder * encoder); static GstFlowReturn gst_amc_video_enc_drain (GstAmcVideoEnc * self); #define BIT_RATE_DEFAULT (2 * 1024 * 1024) -#define I_FRAME_INTERVAL_DEFAULT 0 +#define I_FRAME_INTERVAL_DEFAULT -1 enum { PROP_0, @@ -181,6 +181,10 @@ create_amc_format (GstAmcVideoEnc * encoder, GstVideoCodecState * input_state, profile_string = gst_structure_get_string (s, "profile"); level_string = gst_structure_get_string (s, "level"); + GST_DEBUG_OBJECT (encoder, + "Creating GstAmcFormat for profile: %s and level: %s", + profile_string, level_string); + if (strcmp (name, "video/mpeg") == 0) { gint mpegversion; @@ -262,7 +266,7 @@ create_amc_format (GstAmcVideoEnc * encoder, GstVideoCodecState * input_state, gst_amc_format_set_int (format, "color-format", color_format, &err); if (err) GST_ELEMENT_WARNING_FROM_ERROR (encoder, err); - stride = GST_ROUND_UP_4 (info->width); /* safe (?) */ + stride = info->width; gst_amc_format_set_int (format, "stride", stride, &err); if (err) GST_ELEMENT_WARNING_FROM_ERROR (encoder, err); @@ -277,14 +281,20 @@ create_amc_format (GstAmcVideoEnc * encoder, GstVideoCodecState * input_state, /* FIXME: Set to any value in AVCProfile* leads to * codec configuration fail */ - /* gst_amc_format_set_int (format, amc_profile.key, 0x40); */ + GST_DEBUG_OBJECT (encoder, "Setting profile-id to %d", amc_profile.id); + gst_amc_format_set_int (format, amc_profile.key, amc_profile.id, &err); + if (err) + GST_ELEMENT_WARNING_FROM_ERROR (encoder, err); } if (level_string) { if (amc_level.id == -1) goto unsupported_level; - /* gst_amc_format_set_int (format, amc_level.key, amc_level.id); */ + GST_DEBUG_OBJECT (encoder, "Setting level-id to %d", amc_level.id); + gst_amc_format_set_int (format, amc_level.key, amc_level.id, &err); + if (err) + GST_ELEMENT_WARNING_FROM_ERROR (encoder, err); } /* On Android N_MR1 and higher, i-frame-interval can be a float value */ @@ -427,7 +437,7 @@ caps_from_amc_format (GstAmcFormat * amc_format) } if (gst_amc_format_get_int (amc_format, "level", &amc_level, NULL)) { - level_string = gst_amc_avc_level_to_string (amc_profile); + level_string = gst_amc_avc_level_to_string (amc_level); if (!level_string) goto unsupported_level; @@ -471,7 +481,8 @@ caps_from_amc_format (GstAmcFormat * amc_format) gst_caps_set_simple (caps, "width", G_TYPE_INT, width, "height", G_TYPE_INT, height, - "framerate", GST_TYPE_FRACTION, fraction_n, fraction_d, NULL); + "framerate", GST_TYPE_FRACTION, fraction_n, fraction_d, + "alignment", G_TYPE_STRING, "au", NULL); g_free (mime); return caps; @@ -565,7 +576,7 @@ gst_amc_video_enc_set_property (GObject * object, guint prop_id, break; case PROP_I_FRAME_INTERVAL: - encoder->i_frame_int = g_value_get_uint (value); + encoder->i_frame_int = g_value_get_int (value); if (codec_active) goto wrong_state; break; @@ -603,7 +614,7 @@ gst_amc_video_enc_get_property (GObject * object, guint prop_id, g_value_set_uint (value, encoder->bitrate); break; case PROP_I_FRAME_INTERVAL: - g_value_set_uint (value, encoder->i_frame_int); + g_value_set_int (value, encoder->i_frame_int); break; case PROP_I_FRAME_INTERVAL_FLOAT: g_value_set_float (value, encoder->i_frame_int); @@ -615,6 +626,12 @@ gst_amc_video_enc_get_property (GObject * object, guint prop_id, GST_OBJECT_UNLOCK (encoder); } +static gboolean +gst_amc_video_enc_transform_meta (G_GNUC_UNUSED GstVideoEncoder * encoder, + G_GNUC_UNUSED GstVideoCodecFrame * frame, G_GNUC_UNUSED GstMeta * meta) +{ + return TRUE; +} static void gst_amc_video_enc_class_init (GstAmcVideoEncClass * klass) @@ -642,6 +659,8 @@ gst_amc_video_enc_class_init (GstAmcVideoEncClass * klass) videoenc_class->handle_frame = GST_DEBUG_FUNCPTR (gst_amc_video_enc_handle_frame); videoenc_class->finish = GST_DEBUG_FUNCPTR (gst_amc_video_enc_finish); + videoenc_class->transform_meta = + GST_DEBUG_FUNCPTR (gst_amc_video_enc_transform_meta); // On Android >= 19, we can set bitrate dynamically // so add the flag so apps can detect it. @@ -653,17 +672,26 @@ gst_amc_video_enc_class_init (GstAmcVideoEncClass * klass) G_MAXINT, BIT_RATE_DEFAULT, dynamic_flag | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + /* + From developer.android.com: + A key describing the frequency of key frames expressed in seconds + between key frames. + This key is used by video encoders. A negative value means no key frames + are requested after the first frame. + A zero value means a stream containing all key frames is requested. + */ g_object_class_install_property (gobject_class, PROP_I_FRAME_INTERVAL, - g_param_spec_uint ("i-frame-interval", "I-frame interval", - "The frequency of I frames expressed in seconds between I frames (0 for automatic)", - 0, G_MAXINT, I_FRAME_INTERVAL_DEFAULT, + g_param_spec_int ("i-frame-interval", "I-frame interval", + "The frequency of I frames expressed in seconds between I frames " + "(-1 for never after first, 0 for every frame)", + -1, G_MAXINT, I_FRAME_INTERVAL_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_I_FRAME_INTERVAL_FLOAT, g_param_spec_float ("i-frame-interval-float", "I-frame interval", "The frequency of I frames expressed in seconds between I frames (0 for automatic). " "Fractional intervals work on Android >= 25", - 0, G_MAXFLOAT, I_FRAME_INTERVAL_DEFAULT, + -1, G_MAXFLOAT, I_FRAME_INTERVAL_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); } @@ -765,10 +793,12 @@ gst_amc_video_enc_change_state (GstElement * element, GstStateChange transition) case GST_STATE_CHANGE_PAUSED_TO_PLAYING: break; case GST_STATE_CHANGE_PAUSED_TO_READY: - self->flushing = TRUE; - gst_amc_codec_flush (self->codec, &err); - if (err) - GST_ELEMENT_WARNING_FROM_ERROR (self, err); + if (!self->flushing) { + self->flushing = TRUE; + gst_amc_codec_flush (self->codec, &err); + if (err) + GST_ELEMENT_WARNING_FROM_ERROR (self, err); + } g_mutex_lock (&self->drain_lock); self->draining = FALSE; g_cond_broadcast (&self->drain_cond); @@ -811,8 +841,8 @@ _find_nearest_frame (GstAmcVideoEnc * self, GstClockTime reference_timestamp) GList *l, *best_l = NULL; GList *finish_frames = NULL; GstVideoCodecFrame *best = NULL; - guint64 best_timestamp = 0; - guint64 best_diff = G_MAXUINT64; + GstClockTime best_timestamp = 0; + GstClockTimeDiff best_diff = G_MAXINT64; BufferIdentification *best_id = NULL; GList *frames; @@ -821,21 +851,24 @@ _find_nearest_frame (GstAmcVideoEnc * self, GstClockTime reference_timestamp) for (l = frames; l; l = l->next) { GstVideoCodecFrame *tmp = l->data; BufferIdentification *id = gst_video_codec_frame_get_user_data (tmp); - guint64 timestamp, diff; + GstClockTime timestamp; + GstClockTimeDiff diff; /* This happens for frames that were just added but * which were not passed to the component yet. Ignore * them here! */ - if (!id) + if (!id) { + GST_WARNING_OBJECT (self, "Found frame without BufferIdentification"); continue; + } timestamp = id->timestamp; - if (timestamp > reference_timestamp) - diff = timestamp - reference_timestamp; - else - diff = reference_timestamp - timestamp; + diff = ABS (GST_CLOCK_DIFF (timestamp, reference_timestamp)); + GST_LOG_OBJECT (self, + "Found frame with ID: %" GST_TIME_FORMAT " and diff: %" GST_TIME_FORMAT, + GST_TIME_ARGS (id->timestamp), GST_TIME_ARGS (diff)); if (best == NULL || diff < best_diff) { best = tmp; @@ -844,6 +877,8 @@ _find_nearest_frame (GstAmcVideoEnc * self, GstClockTime reference_timestamp) best_l = l; best_id = id; + GST_DEBUG_OBJECT (self, "We got a best frame"); + /* For frames without timestamp we simply take the first frame */ if ((reference_timestamp == 0 && !GST_CLOCK_TIME_IS_VALID (timestamp)) || diff == 0) @@ -877,6 +912,8 @@ _find_nearest_frame (GstAmcVideoEnc * self, GstClockTime reference_timestamp) if (finish_frames) { g_warning ("%s: Too old frames, bug in encoder -- please file a bug", GST_ELEMENT_NAME (self)); + GST_ERROR_OBJECT (self, + "Too old frames, bug in encoder -- please file a bug"); for (l = finish_frames; l; l = l->next) { gst_video_encoder_finish_frame (GST_VIDEO_ENCODER (self), l->data); } @@ -898,6 +935,8 @@ _find_nearest_frame (GstAmcVideoEnc * self, GstClockTime reference_timestamp) g_list_foreach (frames, (GFunc) gst_video_codec_frame_unref, NULL); g_list_free (frames); + GST_DEBUG_OBJECT (self, "Returning %p", best); + return best; } @@ -957,8 +996,11 @@ gst_amc_video_enc_fill_buffer (GstAmcVideoEnc * self, GstBuffer * inbuf, * then we can use state->info safely */ GstVideoInfo *info = &input_state->info; - if (buffer_info->size < self->color_format_info.frame_size) + if (buffer_info->size < self->color_format_info.frame_size) { + GST_WARNING_OBJECT (self, "Buffer size too small for frame (%u < %u)", + buffer_info->size, self->color_format_info.frame_size); return FALSE; + } return gst_amc_color_format_copy (&self->color_format_info, outbuf, buffer_info, info, inbuf, COLOR_FORMAT_COPY_IN); @@ -989,9 +1031,20 @@ gst_amc_video_enc_handle_output_frame (GstAmcVideoEnc * self, GST_BUFFER_PTS (out_buf) = gst_util_uint64_scale (buffer_info->presentation_time_us, GST_USECOND, 1); + GST_BUFFER_DTS (out_buf) = GST_BUFFER_PTS (out_buf); + + if (!(buffer_info->flags & BUFFER_FLAG_PARTIAL_FRAME)) { + GST_DEBUG_OBJECT (self, "Frame is complete"); + GST_BUFFER_FLAG_SET (out_buf, GST_BUFFER_FLAG_MARKER); + } if (frame) { frame->output_buffer = out_buf; + if (buffer_info->flags & BUFFER_FLAG_KEY_FRAME) { + GST_DEBUG_OBJECT (self, "Frame is sync frame!"); + GST_VIDEO_CODEC_FRAME_SET_SYNC_POINT (frame); + } + flow_ret = gst_video_encoder_finish_frame (encoder, frame); } else { /* This sometimes happens at EOS or if the input is not properly framed, @@ -1027,13 +1080,12 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) GST_VIDEO_ENCODER_STREAM_LOCK (self); retry: - GST_DEBUG_OBJECT (self, "Waiting for available output buffer"); + GST_LOG_OBJECT (self, "Waiting for available output buffer"); GST_VIDEO_ENCODER_STREAM_UNLOCK (self); - /* Wait at most 100ms here, some codecs don't fail dequeueing if + /* Wait at most 1s here, some codecs don't fail dequeueing if * the codec is flushing, causing deadlocks during shutdown */ - idx = - gst_amc_codec_dequeue_output_buffer (self->codec, &buffer_info, 100000, - &err); + idx = gst_amc_codec_dequeue_output_buffer (self->codec, &buffer_info, + G_USEC_PER_SEC, &err); GST_VIDEO_ENCODER_STREAM_LOCK (self); /*} */ @@ -1050,7 +1102,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) GstAmcFormat *format; gchar *format_string; - GST_DEBUG_OBJECT (self, "Output format has changed"); + GST_INFO_OBJECT (self, "Output format has changed"); format = (idx == INFO_OUTPUT_FORMAT_CHANGED) ? gst_amc_codec_get_output_format (self->codec, @@ -1074,7 +1126,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) gst_amc_format_free (format); goto format_error; } - GST_DEBUG_OBJECT (self, "Got new output format: %s", format_string); + GST_INFO_OBJECT (self, "Got new output format: %s", format_string); g_free (format_string); if (!gst_amc_video_enc_set_src_caps (self, format)) { @@ -1096,7 +1148,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) g_assert_not_reached (); break; case INFO_TRY_AGAIN_LATER: - GST_DEBUG_OBJECT (self, "Dequeueing output buffer timed out"); + GST_LOG_OBJECT (self, "Dequeueing output buffer timed out"); goto retry; break; case G_MININT: @@ -1112,7 +1164,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) } process_buffer: - GST_DEBUG_OBJECT (self, + GST_LOG_OBJECT (self, "Got output buffer at index %d: size %d time %" G_GINT64_FORMAT " flags 0x%08x", idx, buffer_info.size, buffer_info.presentation_time_us, buffer_info.flags); @@ -1210,7 +1262,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) g_mutex_unlock (&self->drain_lock); GST_VIDEO_ENCODER_STREAM_LOCK (self); } else { - GST_DEBUG_OBJECT (self, "Finished frame: %s", gst_flow_get_name (flow_ret)); + GST_LOG_OBJECT (self, "Finished frame: %s", gst_flow_get_name (flow_ret)); } self->downstream_flow_ret = flow_ret; @@ -1268,7 +1320,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) } flushing: { - GST_DEBUG_OBJECT (self, "Flushing -- stopping task"); + GST_INFO_OBJECT (self, "Flushing -- stopping task"); gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self)); self->downstream_flow_ret = GST_FLOW_FLUSHING; GST_VIDEO_ENCODER_STREAM_UNLOCK (self); @@ -1278,7 +1330,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) flow_error: { if (flow_ret == GST_FLOW_EOS) { - GST_DEBUG_OBJECT (self, "EOS"); + GST_ERROR_OBJECT (self, "EOS"); gst_pad_push_event (GST_VIDEO_ENCODER_SRC_PAD (self), gst_event_new_eos ()); gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self)); @@ -1288,6 +1340,7 @@ gst_amc_video_enc_loop (GstAmcVideoEnc * self) gst_event_new_eos ()); gst_pad_pause_task (GST_VIDEO_ENCODER_SRC_PAD (self)); } + self->downstream_flow_ret = flow_ret; GST_VIDEO_ENCODER_STREAM_UNLOCK (self); g_mutex_lock (&self->drain_lock); self->draining = FALSE; @@ -1552,7 +1605,7 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, self = GST_AMC_VIDEO_ENC (encoder); - GST_DEBUG_OBJECT (self, "Handling frame"); + GST_LOG_OBJECT (self, "Handling frame"); if (!self->started) { GST_ERROR_OBJECT (self, "Codec not started yet"); @@ -1571,7 +1624,7 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, if (GST_VIDEO_CODEC_FRAME_IS_FORCE_KEYFRAME (frame)) { if (gst_amc_codec_request_key_frame (self->codec, &err)) { - GST_DEBUG_OBJECT (self, "Passed keyframe request to MediaCodec"); + GST_INFO_OBJECT (self, "Passed keyframe request to MediaCodec"); } if (err) { GST_ELEMENT_WARNING_FROM_ERROR (self, err); @@ -1612,12 +1665,15 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, } if (self->flushing) { + GST_INFO_OBJECT (self, "We are flushing"); memset (&buffer_info, 0, sizeof (buffer_info)); gst_amc_codec_queue_input_buffer (self->codec, idx, &buffer_info, NULL); goto flushing; } if (self->downstream_flow_ret != GST_FLOW_OK) { + GST_INFO_OBJECT (self, "Downstream returned %s", + gst_flow_get_name (self->downstream_flow_ret)); memset (&buffer_info, 0, sizeof (buffer_info)); gst_amc_codec_queue_input_buffer (self->codec, idx, &buffer_info, &err); if (err && !self->flushing) @@ -1631,6 +1687,8 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, /* Copy the buffer content in chunks of size as requested * by the port */ buf = gst_amc_codec_get_input_buffer (self->codec, idx, &err); + GST_LOG_OBJECT (self, "Got input buffer: %" GST_PTR_FORMAT, buf); + if (err) goto failed_to_get_input_buffer; else if (!buf) @@ -1644,6 +1702,7 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, if (!gst_amc_video_enc_fill_buffer (self, frame->input_buffer, buf, &buffer_info)) { + GST_ERROR_OBJECT (self, "Could not fill buffer!"); memset (&buffer_info, 0, sizeof (buffer_info)); gst_amc_codec_queue_input_buffer (self->codec, idx, &buffer_info, &err); if (err && !self->flushing) @@ -1666,12 +1725,16 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, self->last_upstream_ts += duration; id = buffer_identification_new (timestamp + timestamp_offset); - if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame)) - buffer_info.flags |= BUFFER_FLAG_SYNC_FRAME; + GST_LOG_OBJECT (self, "Created buffer id with timestamp: %" + GST_TIME_FORMAT, GST_TIME_ARGS (id->timestamp)); + if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame)) { + GST_DEBUG_OBJECT (self, "Frame is sync point"); + buffer_info.flags |= BUFFER_FLAG_KEY_FRAME; + } gst_video_codec_frame_set_user_data (frame, id, (GDestroyNotify) buffer_identification_free); - GST_DEBUG_OBJECT (self, + GST_LOG_OBJECT (self, "Queueing buffer %d: size %d time %" G_GINT64_FORMAT " flags 0x%08x", idx, buffer_info.size, buffer_info.presentation_time_us, buffer_info.flags); @@ -1705,6 +1768,8 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, } got_null_input_buffer: { + GST_ERROR_OBJECT (self, "got_null_input_buffer"); + GST_ELEMENT_ERROR (self, LIBRARY, SETTINGS, (NULL), ("Got no input buffer")); gst_video_codec_frame_unref (frame); @@ -1712,6 +1777,7 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, } buffer_fill_error: { + GST_ERROR_OBJECT (self, "buffer_fill_error"); GST_ELEMENT_ERROR (self, RESOURCE, WRITE, (NULL), ("Failed to write input into the amc buffer(write %dB to a %" G_GSIZE_FORMAT "B buffer)", self->color_format_info.frame_size, @@ -1733,6 +1799,7 @@ gst_amc_video_enc_handle_frame (GstVideoEncoder * encoder, } flushing: { + GST_ERROR_OBJECT (self, "flushing"); GST_DEBUG_OBJECT (self, "Flushing -- returning FLUSHING"); gst_video_codec_frame_unref (frame); return GST_FLOW_FLUSHING; diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c b/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c new file mode 100644 index 00000000000..d298a5dc079 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.c @@ -0,0 +1,807 @@ +/* Gstreamer + * Copyright (C) 2022 Pexip (https://pexip.com/) + * @author: Tulio Beloqui + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "gstamdeviceprovider.h" + +#include "gstjniutils.h" + +struct _GstAmDeviceProvider +{ + GstDeviceProvider parent; + + jobject audioDeviceCB; +}; + +struct _GstAmDevice +{ + GstDevice parent; + + gint device_id; +}; + +enum +{ + ADDED_DEVICES, + REMOVED_DEVICES, +}; + +G_DEFINE_TYPE (GstAmDeviceProvider, gst_am_device_provider, + GST_TYPE_DEVICE_PROVIDER); + +G_DEFINE_TYPE (GstAmDevice, gst_am_device, GST_TYPE_DEVICE); + +GST_DEVICE_PROVIDER_REGISTER_DEFINE (amdeviceprovider, "amdeviceprovider", + GST_RANK_PRIMARY, GST_TYPE_AM_DEVICE_PROVIDER); + +GST_DEBUG_CATEGORY_STATIC (gst_am_device_provider_debug); +#define GST_CAT_DEFAULT (gst_am_device_provider_debug) +#define device_provider_parent_class gst_am_device_provider_parent_class + +#define GET_DEVICES_ALL 0x00000003 + +/* from: https://developer.android.com/reference/android/media/AudioDeviceInfo */ +typedef enum +{ + TYPE_AUX_LINE = 19, + TYPE_BLE_BROADCAST = 30, + TYPE_BLE_HEADSET = 26, + TYPE_BLE_SPEAKER = 27, + TYPE_BLUETOOTH_A2DP = 8, + TYPE_BLUETOOTH_SCO = 7, + TYPE_BUILTIN_EARPIECE = 1, + TYPE_BUILTIN_MIC = 15, + TYPE_BUILTIN_SPEAKER = 2, + TYPE_BUILTIN_SPEAKER_SAFE = 24, + TYPE_BUS = 21, + TYPE_DOCK = 13, + TYPE_FM = 14, + TYPE_FM_TUNER = 16, + TYPE_HDMI = 9, + TYPE_HDMI_ARC = 10, + TYPE_HDMI_EARC = 29, + TYPE_HEARING_AID = 23, + TYPE_IP = 20, + TYPE_LINE_ANALOG = 5, + TYPE_LINE_DIGITAL = 6, + TYPE_REMOTE_SUBMIX = 25, + TYPE_TELEPHONY = 18, + TYPE_TV_TUNER = 17, + TYPE_UNKNOWN = 0, + TYPE_USB_ACCESSORY = 12, + TYPE_USB_DEVICE = 11, + TYPE_USB_HEADSET = 22, + TYPE_WIRED_HEADPHONES = 4, + TYPE_WIRED_HEADSET = 3, +} AudioInfo_DeviceType; + +static jobject +jni_get_global_context (JNIEnv * env) +{ + jclass activityThread; + jmethodID currentActivityThread; + jobject activity; + jmethodID getApplication; + jobject ctx; + + activityThread = (*env)->FindClass (env, "android/app/ActivityThread"); + currentActivityThread = + (*env)->GetStaticMethodID (env, activityThread, "currentActivityThread", + "()Landroid/app/ActivityThread;"); + activity = (*env)->CallStaticObjectMethod (env, activityThread, + currentActivityThread); + getApplication = + (*env)->GetMethodID (env, activityThread, "getApplication", + "()Landroid/app/Application;"); + + ctx = (*env)->CallObjectMethod (env, activity, getApplication); + return ctx; +} + +static jobject +jni_get_AudioManager_Instance (JNIEnv * env, jobject context) +{ + jclass contextClass; + jfieldID audioServiceField; + jstring audioService; + jmethodID getSystemServiceID; + jobject audioManager; + + contextClass = (*env)->FindClass (env, "android/content/Context"); + audioServiceField = + (*env)->GetStaticFieldID (env, contextClass, "AUDIO_SERVICE", + "Ljava/lang/String;"); + audioService = + (jstring) (*env)->GetStaticObjectField (env, contextClass, + audioServiceField); + getSystemServiceID = + (*env)->GetMethodID (env, contextClass, "getSystemService", + "(Ljava/lang/String;)Ljava/lang/Object;"); + + audioManager = (*env)->CallObjectMethod (env, context, + getSystemServiceID, audioService); + + g_assert (audioManager != NULL); + + return audioManager; +} + +static gint +jni_AudioInfo_getId (JNIEnv * env, jobject audioInfo) +{ + jclass audioInfoClass; + jmethodID getId; + + audioInfoClass = (*env)->FindClass (env, "android/media/AudioDeviceInfo"); + getId = (*env)->GetMethodID (env, audioInfoClass, "getId", "()I"); + + return (gint) (*env)->CallIntMethod (env, audioInfo, getId); +} + +static gboolean +jni_AudioInfo_isSink (JNIEnv * env, jobject audioInfo) +{ + jclass audioInfoClass; + jmethodID isSink; + + audioInfoClass = (*env)->FindClass (env, "android/media/AudioDeviceInfo"); + isSink = (*env)->GetMethodID (env, audioInfoClass, "isSink", "()Z"); + + return (gint) (*env)->CallBooleanMethod (env, audioInfo, isSink); +} + +static gboolean +jni_AudioInfo_isSource (JNIEnv * env, jobject audioInfo) +{ + jclass audioInfoClass; + jmethodID isSource; + + audioInfoClass = (*env)->FindClass (env, "android/media/AudioDeviceInfo"); + isSource = (*env)->GetMethodID (env, audioInfoClass, "isSource", "()Z"); + + return (gint) (*env)->CallBooleanMethod (env, audioInfo, isSource); +} + +static gchar * +jni_String_GetStringUTFChars (JNIEnv * env, jstring str) +{ + gchar *value = NULL; + const char *utf; + + utf = (*env)->GetStringUTFChars (env, str, NULL); + if (utf) { + value = g_strdup (utf); + (*env)->ReleaseStringUTFChars (env, str, utf); + } + + return value; +} + +static jstring +jni_Object_toString (JNIEnv * env, jobject obj) +{ + jclass objClass; + jmethodID toString; + + objClass = (*env)->FindClass (env, "java/lang/Object"); + toString = + (*env)->GetMethodID (env, objClass, "toString", "()Ljava/lang/String;"); + + return (*env)->CallObjectMethod (env, obj, toString); +} + +static gchar * +jni_AudioInfo_getProductName (JNIEnv * env, jobject audioInfo) +{ + jclass audioInfoClass; + + jmethodID getProductName; + + jobject productNameCharSeq; + jobject productNameString; + + audioInfoClass = (*env)->FindClass (env, "android/media/AudioDeviceInfo"); + + getProductName = + (*env)->GetMethodID (env, audioInfoClass, "getProductName", + "()Ljava/lang/CharSequence;"); + + productNameCharSeq = + (*env)->CallObjectMethod (env, audioInfo, getProductName); + productNameString = jni_Object_toString (env, productNameCharSeq); + + return jni_String_GetStringUTFChars (env, productNameString); +} + +static void +gst_am_device_provider_remove_device (GstAmDeviceProvider * amprovider, + JNIEnv * env, jobject audioInfo) +{ + GstDeviceProvider *provider = GST_DEVICE_PROVIDER_CAST (amprovider); + GstDevice *device = NULL; + GList *item; + gint id_to_remove; + + id_to_remove = jni_AudioInfo_getId (env, audioInfo); + + GST_OBJECT_LOCK (provider); + for (item = provider->devices; item; item = item->next) { + device = item->data; + gint device_id; + + g_object_get (device, "device-id", &device_id, NULL); + if (id_to_remove == device_id) { + gst_object_ref (device); + break; + } + + device = NULL; + } + GST_OBJECT_UNLOCK (provider); + + if (device) { + GST_DEBUG_OBJECT (provider, "Removing %" GST_PTR_FORMAT, device); + gst_device_provider_device_remove (provider, GST_DEVICE (device)); + g_object_unref (device); + } +} + +static gchar * +jni_get_AudioInfo_DeviceType_Name (AudioInfo_DeviceType deviceType) +{ + switch (deviceType) { + case TYPE_BLE_BROADCAST: + case TYPE_BLE_HEADSET: + case TYPE_BLE_SPEAKER: + case TYPE_BLUETOOTH_A2DP: + case TYPE_BLUETOOTH_SCO: + return g_strdup ("Bluetooth"); + + case TYPE_BUILTIN_EARPIECE: + case TYPE_BUILTIN_MIC: + case TYPE_BUILTIN_SPEAKER: + case TYPE_BUILTIN_SPEAKER_SAFE: + return g_strdup ("Built-in"); + + case TYPE_HDMI: + case TYPE_HDMI_ARC: + case TYPE_HDMI_EARC: + return g_strdup ("HDMI"); + + case TYPE_USB_ACCESSORY: + case TYPE_USB_DEVICE: + case TYPE_USB_HEADSET: + return g_strdup ("USB"); + + case TYPE_WIRED_HEADPHONES: + case TYPE_WIRED_HEADSET: + return g_strdup ("Wired"); + + default: + break; + } + + return NULL; +} + +static AudioInfo_DeviceType +jni_AudioInfo_getDeviceType (JNIEnv * env, jobject audioInfo) +{ + jclass audioInfoClass; + jmethodID getType; + + audioInfoClass = (*env)->FindClass (env, "android/media/AudioDeviceInfo"); + getType = (*env)->GetMethodID (env, audioInfoClass, "getType", "()I"); + + return (AudioInfo_DeviceType) (*env)->CallIntMethod (env, audioInfo, getType); +} + + +static GstDevice * +gst_am_device_from_audio_info (JNIEnv * env, jobject audioInfo) +{ + gint id; + gchar *product_name; + gchar *display_name; + gchar *type_name; + gboolean is_sink; + gboolean is_source; + AudioInfo_DeviceType deviceType; + GstDevice *device; + GstCaps *caps; + const gchar *device_class = NULL; + + is_sink = jni_AudioInfo_isSink (env, audioInfo); + is_source = jni_AudioInfo_isSource (env, audioInfo); + + if (!is_sink && !is_source) + return NULL; + + deviceType = jni_AudioInfo_getDeviceType (env, audioInfo); + type_name = jni_get_AudioInfo_DeviceType_Name (deviceType); + + /* filter the enumerated devices by supported type */ + if (!type_name) + return NULL; + + id = jni_AudioInfo_getId (env, audioInfo); + product_name = jni_AudioInfo_getProductName (env, audioInfo); + display_name = g_strdup_printf ("%s (%s)", product_name, type_name); + // FIXME: call getAudioProfiles() on the audioInfo and transform to caps + caps = gst_caps_new_empty_simple ("audio/x-raw"); + + g_free (type_name); + g_free (product_name); + + if (is_source) + device_class = "Audio/Source"; + if (is_sink) + device_class = "Audio/Sink"; + + device = g_object_new (GST_TYPE_AM_DEVICE, // + "device-id", id, // + "device-class", device_class, // + "display-name", display_name, // + "caps", caps, // + NULL); + + return GST_DEVICE_CAST (device); +} + +static void +gst_am_device_provider_add_device (GstAmDeviceProvider * provider, JNIEnv * env, + jobject audioInfo) +{ + GstDevice *device = gst_am_device_from_audio_info (env, audioInfo); + + if (device) { + GST_DEBUG_OBJECT (provider, "Adding %" GST_PTR_FORMAT, device); + gst_device_provider_device_add (GST_DEVICE_PROVIDER_CAST (provider), + device); + } +} + +static void +gst_am_device_provider_on_audio_devices_changed (GstAmDeviceProvider * provider, + JNIEnv * env, jobjectArray audioDevices, int action_type) +{ + jsize i; + jsize numAudioDevices; + + numAudioDevices = (*env)->GetArrayLength (env, audioDevices); + GST_DEBUG_OBJECT (provider, "Audio devices %s (%d)", + action_type == ADDED_DEVICES ? "added" : "removed", numAudioDevices); + + for (i = 0; i < numAudioDevices; i++) { + jobject audioInfo; + audioInfo = (*env)->GetObjectArrayElement (env, audioDevices, i); + + if (action_type == ADDED_DEVICES) + gst_am_device_provider_add_device (provider, env, audioInfo); + else if (action_type == REMOVED_DEVICES) + gst_am_device_provider_remove_device (provider, env, audioInfo); + } +} + +static void +gst_am_device_provider_on_audio_devices_added (JNIEnv * env, jobject thiz, + long long context, jobjectArray addedDevices) +{ + GstAmDeviceProvider *provider = + GST_AM_DEVICE_PROVIDER_CAST (JLONG_TO_GPOINTER (context)); + + gst_am_device_provider_on_audio_devices_changed (provider, env, + addedDevices, ADDED_DEVICES); +} + +static void +gst_am_device_provider_on_audio_devices_removed (JNIEnv * env, + jobject thiz, long long context, jobjectArray removedDevices) +{ + GstAmDeviceProvider *provider = + GST_AM_DEVICE_PROVIDER_CAST (JLONG_TO_GPOINTER (context)); + + gst_am_device_provider_on_audio_devices_changed (provider, env, + removedDevices, REMOVED_DEVICES); +} + +static jobject +gst_am_device_provider_create_callback (JNIEnv * env) +{ + jclass class; + jobject callback = NULL; + + jmethodID constructor_id; + + class = + gst_amc_jni_get_application_class (env, + "org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback", NULL); + if (!class) { + GST_ERROR ("Failed to load GstAmAudioDeviceCallback class"); + return NULL; + } + + constructor_id = + gst_amc_jni_get_method_id (env, NULL, class, "", "()V"); + if (!constructor_id) { + GST_ERROR ("Failed to get method id for constructor"); + goto done; + } + + callback = gst_amc_jni_new_object (env, NULL, TRUE, class, constructor_id); + +done: + if (class) + gst_amc_jni_object_unref (env, class); + + return callback; +} + +static void +gst_am_device_provider_callback_set_context (JNIEnv * env, + jobject audioDeviceCallback, jlong context) +{ + jclass class; + jmethodID setContext; + + class = + gst_amc_jni_get_application_class (env, + "org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback", NULL); + if (!class) { + GST_ERROR ("Failed to load GstAmAudioDeviceCallback class"); + return; + } + + setContext = + gst_amc_jni_get_method_id (env, NULL, class, "setContext", "(J)V"); + + /* call setContext on GstAmAudioDeviceCallback */ + if (!gst_amc_jni_call_void_method (env, NULL, audioDeviceCallback, setContext, + context)) { + GST_ERROR ("setContext call failed"); + } + + if (class) + gst_amc_jni_object_unref (env, class); +} + +static void +jni_AudioManager_registerAudioDeviceCallback (JNIEnv * env, + jobject audioManager, jobject audioDeviceCallback) +{ + jclass audioManagerClass; + jmethodID registerAudioDeviceCallback; + + audioManagerClass = (*env)->FindClass (env, "android/media/AudioManager"); + registerAudioDeviceCallback = + (*env)->GetMethodID (env, audioManagerClass, + "registerAudioDeviceCallback", + "(Landroid/media/AudioDeviceCallback;Landroid/os/Handler;)V"); + + (*env)->CallVoidMethod (env, audioManager, registerAudioDeviceCallback, + audioDeviceCallback, NULL); +} + +static void +jni_AudioManager_unregisterAudioDeviceCallback (JNIEnv * env, + jobject audioManager, jobject audioDeviceCallback) +{ + jclass audioManagerClass; + jmethodID unregisterAudioDeviceCallback; + + audioManagerClass = (*env)->FindClass (env, "android/media/AudioManager"); + unregisterAudioDeviceCallback = + (*env)->GetMethodID (env, audioManagerClass, + "unregisterAudioDeviceCallback", + "(Landroid/media/AudioDeviceCallback;)V"); + + (*env)->CallVoidMethod (env, audioManager, unregisterAudioDeviceCallback, + audioDeviceCallback); +} + +static jobjectArray +jni_AudioManager_getDevices (JNIEnv * env, jobject audioManager, + gint deviceType) +{ + jobjectArray audioDevices; // AudioDeviceInfo[] + jclass audioManagerClass; + jmethodID getDevicesID; + + audioManagerClass = (*env)->FindClass (env, "android/media/AudioManager"); + getDevicesID = (*env)->GetMethodID (env, audioManagerClass, "getDevices", + "(I)[Landroid/media/AudioDeviceInfo;"); + audioDevices = + (*env)->CallObjectMethod (env, audioManager, getDevicesID, deviceType); + + return audioDevices; +} + +static GList * +gst_am_device_provider_probe (GstDeviceProvider * object) +{ + GList *devices = NULL; + JNIEnv *env; + jobject ctx; + jobject audioManager; + jsize i, numAudioDevices; + jobjectArray audioDevices; + + env = gst_amc_jni_get_env (); + ctx = jni_get_global_context (env); + if (!ctx) { + GST_ERROR ("Can't get an instance of global context"); + return NULL; + } + + audioManager = jni_get_AudioManager_Instance (env, ctx); + if (!audioManager) { + GST_ERROR ("Can't get an instance of AudioManager"); + return NULL; + } + + audioDevices = + jni_AudioManager_getDevices (env, audioManager, GET_DEVICES_ALL); + if (!audioDevices) { + GST_ERROR ("No audio devices found!"); + return NULL; + } + + numAudioDevices = (*env)->GetArrayLength (env, audioDevices); + GST_DEBUG ("numAudioDevices: %u", numAudioDevices); + for (i = 0; i < numAudioDevices; i++) { + jobject audioInfo; + GstDevice *device; + + audioInfo = (*env)->GetObjectArrayElement (env, audioDevices, i); + device = gst_am_device_from_audio_info (env, audioInfo); + + if (device) { + gst_object_ref_sink (device); + devices = g_list_append (devices, device); + } + } + + return devices; +} + +static void +gst_am_device_provider_populate_devices (GstDeviceProvider * provider) +{ + GList *devices, *it; + + devices = gst_am_device_provider_probe (provider); + + for (it = devices; it != NULL; it = g_list_next (it)) { + GstDevice *device = GST_DEVICE_CAST (it->data); + if (device) + gst_device_provider_device_add (provider, device); + } + + g_list_free_full (devices, gst_object_unref); +} + +static gboolean +gst_am_device_provider_start (GstDeviceProvider * object) +{ + GstAmDeviceProvider *provider; + JNIEnv *env; + jobject ctx; + jobject audioDeviceCallback; + jobject audioManager; + + provider = GST_AM_DEVICE_PROVIDER_CAST (object); + if (provider->audioDeviceCB != NULL) { + GST_ERROR ("Already started!"); + return FALSE; + } + + GST_DEBUG_OBJECT (provider, "Starting..."); + + gst_am_device_provider_populate_devices (object); + + env = gst_amc_jni_get_env (); + audioDeviceCallback = gst_am_device_provider_create_callback (env); + if (!audioDeviceCallback) { + return FALSE; + } + + gst_am_device_provider_callback_set_context (env, audioDeviceCallback, + GPOINTER_TO_JLONG (provider)); + + + ctx = jni_get_global_context (env); + if (!ctx) { + GST_ERROR ("Can't get an instance of global context"); + return FALSE; + } + + audioManager = jni_get_AudioManager_Instance (env, ctx); + if (!audioManager) { + GST_ERROR ("Can't get an instance of AudioManager"); + return FALSE; + } + + provider->audioDeviceCB = audioDeviceCallback; + jni_AudioManager_registerAudioDeviceCallback (env, audioManager, + audioDeviceCallback); + + return TRUE; +} + +static void +gst_am_device_provider_stop (GstDeviceProvider * object) +{ + GstAmDeviceProvider *provider; + JNIEnv *env; + jobject ctx; + jobject audioManager; + + provider = GST_AM_DEVICE_PROVIDER_CAST (object); + if (provider->audioDeviceCB == NULL) { + GST_ERROR ("No callback set"); + return; + } + + GST_DEBUG_OBJECT (provider, "Stopping..."); + + env = gst_amc_jni_get_env (); + ctx = jni_get_global_context (env); + if (!ctx) { + GST_ERROR ("Can't get an instance of global context"); + return; + } + + audioManager = jni_get_AudioManager_Instance (env, ctx); + if (!audioManager) { + GST_ERROR ("Can't get an instance of AudioManager"); + return; + } + + gst_am_device_provider_callback_set_context (env, provider->audioDeviceCB, 0); + + jni_AudioManager_unregisterAudioDeviceCallback (env, audioManager, + provider->audioDeviceCB); + gst_amc_jni_object_unref (env, provider->audioDeviceCB); + provider->audioDeviceCB = NULL; +} + +static void +gst_am_device_provider_init (GstAmDeviceProvider * provider) +{ +} + +static void +gst_am_device_provider_finalize (GObject * object) +{ + GstAmDeviceProvider *provider = GST_AM_DEVICE_PROVIDER_CAST (object); + (void) provider; + + G_OBJECT_CLASS (device_provider_parent_class)->finalize (object); +} + +static void +gst_am_device_provider_register_jni_native_methods () +{ + jclass class; + JNIEnv *env; + + JNINativeMethod native_methods[] = { + {"native_onAudioDevicesAdded", "(J[Landroid/media/AudioDeviceInfo;)V", + (void *) gst_am_device_provider_on_audio_devices_added}, + {"native_onAudioDevicesRemoved", "(J[Landroid/media/AudioDeviceInfo;)V", + (void *) gst_am_device_provider_on_audio_devices_removed}, + }; + + env = gst_amc_jni_get_env (); + class = + gst_amc_jni_get_application_class (env, + "org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback", NULL); + if (!class) { + GST_ERROR ("Failed to load GstAmAudioDeviceCallback class"); + return; + } + + (*env)->RegisterNatives (env, class, + (const JNINativeMethod *) &native_methods, G_N_ELEMENTS (native_methods)); + if ((*env)->ExceptionCheck (env)) { + GST_ERROR ("Failed to register native methods"); + } + + gst_amc_jni_object_unref (env, class); +} + +static void +gst_am_device_provider_class_init (GstAmDeviceProviderClass * klass) +{ + GstDeviceProviderClass *dm_class = GST_DEVICE_PROVIDER_CLASS (klass); + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + dm_class->probe = gst_am_device_provider_probe; + dm_class->start = gst_am_device_provider_start; + dm_class->stop = gst_am_device_provider_stop; + + gobject_class->finalize = gst_am_device_provider_finalize; + + gst_device_provider_class_set_static_metadata (dm_class, + "Android Audio Device Provider", "Sink/Source/Audio", + "List and monitor Android audio devices", + "Tulio Beloqui "); + + GST_DEBUG_CATEGORY_INIT (gst_am_device_provider_debug, + "amdeviceprovider", 0, "Android Audio Device Provider"); + + gst_am_device_provider_register_jni_native_methods (); +} + +enum +{ + PROP_DEVICE_ID = 1, +}; + +static void +gst_am_device_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstAmDevice *device = GST_AM_DEVICE_CAST (object); + + switch (prop_id) { + case PROP_DEVICE_ID: + g_value_set_int (value, device->device_id); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_am_device_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstAmDevice *device = GST_AM_DEVICE_CAST (object); + + switch (prop_id) { + case PROP_DEVICE_ID: + device->device_id = g_value_get_int (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + + +static void +gst_am_device_class_init (GstAmDeviceClass * klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->get_property = gst_am_device_get_property; + object_class->set_property = gst_am_device_set_property; + + g_object_class_install_property (object_class, PROP_DEVICE_ID, + g_param_spec_int ("device-id", "Audio Device ID", + "The audio device id", -1, G_MAXINT, -1, + G_PARAM_STATIC_STRINGS | G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY)); +} + +static void +gst_am_device_init (GstAmDevice * device) +{ +} diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.h b/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.h new file mode 100644 index 00000000000..9b43c608c1d --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/androidmedia/gstamdeviceprovider.h @@ -0,0 +1,44 @@ +/* Gstreamer + * Copyright (C) 2022 Pexip (https://pexip.com/) + * @author: Tulio Beloqui + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_AM_DEVICE_PROVIDER_H__ +#define __GST_AM_DEVICE_PROVIDER_H__ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_AM_DEVICE_PROVIDER (gst_am_device_provider_get_type ()) +#define GST_AM_DEVICE_PROVIDER_CAST(obj) ((GstAmDeviceProvider *)(obj)) + +#define GST_TYPE_AM_DEVICE (gst_am_device_get_type ()) +#define GST_AM_DEVICE_CAST(obj) ((GstAmDevice *)(obj)) + +G_DECLARE_FINAL_TYPE (GstAmDevice, gst_am_device, GST, AM_DEVICE, + GstDevice) + +G_DECLARE_FINAL_TYPE (GstAmDeviceProvider, gst_am_device_provider, GST, + AM_DEVICE_PROVIDER, GstDeviceProvider) + +GST_DEVICE_PROVIDER_REGISTER_DECLARE (amdeviceprovider); + +G_END_DECLS + +#endif /* __GST_AM_DEVICE_PROVIDER_H__ */ diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/jni/gstamc-codec-jni.c b/subprojects/gst-plugins-bad/sys/androidmedia/jni/gstamc-codec-jni.c index dcc1314f64e..9e0c6288ab6 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/jni/gstamc-codec-jni.c +++ b/subprojects/gst-plugins-bad/sys/androidmedia/jni/gstamc-codec-jni.c @@ -586,7 +586,9 @@ gst_amc_codec_configure (GstAmcCodec * codec, GstAmcFormat * format, env = gst_amc_jni_get_env (); if (surface) { - g_object_unref (codec->surface); + if (codec->surface) + g_object_unref (codec->surface); + codec->surface = gst_amc_surface_new ((GstAmcSurfaceTextureJNI *) surface, err); if (!codec->surface) diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/meson.build b/subprojects/gst-plugins-bad/sys/androidmedia/meson.build index 7199faea2f7..de00294709c 100644 --- a/subprojects/gst-plugins-bad/sys/androidmedia/meson.build +++ b/subprojects/gst-plugins-bad/sys/androidmedia/meson.build @@ -74,7 +74,29 @@ else plugin_name = 'gstandroidmedia' endif -if have_jni_h or have_mlsdk +if have_jni_h + androidmedia_java_sources += [ + 'org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback.java', + ] + androidmedia_sources += [ + 'gstamdeviceprovider.c', + ] +endif + +have_ndkcamera = cc.has_header('camera/NdkCameraError.h', required : false) +if have_ndkcamera + androidmedia_sources += [ + 'gstacamdeviceprovider.c', + 'gstahc2src.c' + ] + + foreach lib : ['camera2ndk', 'mediandk'] + dep = cc.find_library(lib, required : true) + extra_deps += dep + endforeach +endif + +if have_jni_h or have_mlsdk or have_ndkcamera gstandroidmedia = library(plugin_name, androidmedia_sources, c_args : [gst_plugins_bad_args, extra_cargs], diff --git a/subprojects/gst-plugins-bad/sys/androidmedia/org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback.java b/subprojects/gst-plugins-bad/sys/androidmedia/org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback.java new file mode 100644 index 00000000000..cb1f1f9e5e8 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/androidmedia/org/freedesktop/gstreamer/androidmedia/GstAmAudioDeviceCallback.java @@ -0,0 +1,47 @@ +/* Gstreamer + * Copyright (C) 2022 Pexip (https://pexip.com/) + * @author: Tulio Beloqui + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +package org.freedesktop.gstreamer.androidmedia; + +import android.media.AudioDeviceInfo; +import android.media.AudioDeviceCallback; + +public class GstAmAudioDeviceCallback extends AudioDeviceCallback +{ + private long context = 0; + + public void setContext(long c) { + context = c; + } + + public synchronized void onAudioDevicesAdded(AudioDeviceInfo[] addedDevices) { + if (context != 0) + native_onAudioDevicesAdded(context, addedDevices); + } + + public synchronized void onAudioDevicesRemoved(AudioDeviceInfo[] removedDevices) { + if (context != 0) + native_onAudioDevicesRemoved(context, removedDevices); + } + + private native void native_onAudioDevicesAdded (long context, AudioDeviceInfo[] addedDevices); + + private native void native_onAudioDevicesRemoved (long context, AudioDeviceInfo[] addedDevices); +} diff --git a/subprojects/gst-plugins-bad/sys/applemedia/avaudiodeviceprovider.h b/subprojects/gst-plugins-bad/sys/applemedia/avaudiodeviceprovider.h new file mode 100644 index 00000000000..b4cd0b2c2ff --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/applemedia/avaudiodeviceprovider.h @@ -0,0 +1,49 @@ +/* Gstreamer + * Copyright (C) 2022 Pexip (https://pexip.com/) + * @author: Tulio Beloqui + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_AV_AUDIO_DEVICE_PROVIDER_H__ +#define __GST_AV_AUDIO_DEVICE_PROVIDER_H__ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_AV_AUDIO_DEVICE_PROVIDER (gst_av_audio_device_provider_get_type ()) +#define GST_AV_AUDIO_DEVICE_PROVIDER_CAST(obj) ((GstAVAudioDeviceProvider *)(obj)) + +#define GST_TYPE_AV_AUDIO_DEVICE (gst_av_audio_device_get_type ()) +#define GST_AV_AUDIO_DEVICE_CAST(obj) ((GstAVAudioDevice *)(obj)) + +G_DECLARE_FINAL_TYPE (GstAVAudioDevice, gst_av_audio_device, GST, AV_AUDIO_DEVICE, + GstDevice) + +G_DECLARE_FINAL_TYPE (GstAVAudioDeviceProvider, gst_av_audio_device_provider, GST, + AV_AUDIO_DEVICE_PROVIDER, GstDeviceProvider) + +GST_DEVICE_PROVIDER_REGISTER_DECLARE (avaudiodeviceprovider); + +G_END_DECLS + + +#endif /* __GST_AV_AUDIO_DEVICE_PROVIDER_H__ */ diff --git a/subprojects/gst-plugins-bad/sys/applemedia/avaudiodeviceprovider.m b/subprojects/gst-plugins-bad/sys/applemedia/avaudiodeviceprovider.m new file mode 100644 index 00000000000..5e15631a779 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/applemedia/avaudiodeviceprovider.m @@ -0,0 +1,241 @@ +/* Gstreamer + * Copyright (C) 2022 Pexip (https://pexip.com/) + * @author: Tulio Beloqui + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#import + +#include "avaudiodeviceprovider.h" + +struct _GstAVAudioDeviceProvider +{ + GstDeviceProvider parent; +}; + +struct _GstAVAudioDevice +{ + GstDevice parent; + + gchar *type; + gchar *uid; +}; + +G_DEFINE_TYPE (GstAVAudioDeviceProvider, gst_av_audio_device_provider, + GST_TYPE_DEVICE_PROVIDER); + +G_DEFINE_TYPE (GstAVAudioDevice, gst_av_audio_device, GST_TYPE_DEVICE); + +GST_DEVICE_PROVIDER_REGISTER_DEFINE (avaudiodeviceprovider, + "avaudiodeviceprovider", GST_RANK_PRIMARY, + GST_TYPE_AV_AUDIO_DEVICE_PROVIDER); + +GST_DEBUG_CATEGORY_STATIC (gst_av_audio_device_provider_debug); +#define GST_CAT_DEFAULT (gst_av_audio_device_provider_debug) + +static GstDevice * +gst_av_audio_device_new (gchar * name, gchar * type, gchar * uid, + const gchar * device_class) +{ + GstAVAudioDevice *device = g_object_new (GST_TYPE_AV_AUDIO_DEVICE, + "display-name", name, + "device-class", device_class, + "type", type, + "uid", uid, + "caps", gst_caps_new_empty_simple ("audio/x-raw"), + NULL); + + return GST_DEVICE_CAST (device); +} + +static GstDevice * +gst_device_from_port_description (AVAudioSessionPortDescription * + portDesc, const gchar * device_class) +{ + gchar *name = g_strdup ([portDesc.portName UTF8String]); + gchar *type = g_strdup ([portDesc.portType UTF8String]); + gchar *uid = g_strdup ([portDesc.UID UTF8String]); + + return gst_av_audio_device_new (name, type, uid, device_class); +} + +static GList * +gst_av_audio_device_provider_probe (GstDeviceProvider * object) +{ + GList *devices = NULL; + AVAudioSession *audioSession =[AVAudioSession sharedInstance]; + + NSString *sessionCategory = AVAudioSessionCategoryPlayAndRecord; + AVAudioSessionCategoryOptions options = + AVAudioSessionCategoryOptionAllowBluetooth | + AVAudioSessionCategoryOptionAllowBluetoothA2DP | + AVAudioSessionCategoryOptionDefaultToSpeaker; + +[audioSession setCategory: sessionCategory withOptions: options error:nil]; +[audioSession setMode: AVAudioSessionModeVoiceChat error:nil]; +[audioSession setActive: YES error:nil]; + + AVAudioSessionRouteDescription *route = audioSession.currentRoute; + + gboolean addedBuiltinSpk = FALSE; + gboolean addedBuiltinRecv = FALSE; + + if (route != nil) { + for (AVAudioSessionPortDescription * outPort in route.outputs) { + if ([outPort.portType isEqualToString:AVAudioSessionPortBuiltInSpeaker]) + addedBuiltinSpk = TRUE; + + if ([outPort.portType isEqualToString:AVAudioSessionPortBuiltInReceiver]) + addedBuiltinRecv = TRUE; + + GstDevice *device = + gst_device_from_port_description (outPort, "Audio/Sink"); + devices = g_list_append (devices, gst_object_ref_sink (device)); + } + } + + for (AVAudioSessionPortDescription * inPort in[audioSession availableInputs]) { + GstDevice *device = + gst_device_from_port_description (inPort, "Audio/Source"); + devices = g_list_append (devices, gst_object_ref_sink (device)); + } + + if (!addedBuiltinSpk) { + gchar *type = g_strdup ([AVAudioSessionPortBuiltInSpeaker UTF8String]); + gchar *name = g_strdup_printf ("iPhone %s", type); + GstDevice *dev = gst_av_audio_device_new (name, type, NULL, "Audio/Sink"); + + devices = g_list_append (devices, gst_object_ref_sink (dev)); + } + + if (!addedBuiltinRecv) { + gchar *type = g_strdup ([AVAudioSessionPortBuiltInReceiver UTF8String]); + gchar *name = g_strdup_printf ("iPhone %s", type); + GstDevice *dev = gst_av_audio_device_new (name, type, NULL, "Audio/Sink"); + + devices = g_list_append (devices, gst_object_ref_sink (dev)); + } + + return devices; +} + +static void +gst_av_audio_device_provider_init (GstAVAudioDeviceProvider * provider) +{ +} + +static void +gst_av_audio_device_provider_class_init (GstAVAudioDeviceProviderClass * klass) +{ + GstDeviceProviderClass *dm_class = GST_DEVICE_PROVIDER_CLASS (klass); + dm_class->probe = gst_av_audio_device_provider_probe; + + gst_device_provider_class_set_static_metadata (dm_class, + "AV Audio Device Provider", "Sink/Source/Audio", + "List and monitor IOS audio devices", "Tulio Beloqui "); + + GST_DEBUG_CATEGORY_INIT (gst_av_audio_device_provider_debug, + "avaudiodeviceprovider", 0, "AV Audio Device Provider for IOS"); +} + +enum +{ + PROP_DEVICE_TYPE = 1, + PROP_DEVICE_UID, +}; + +static void +gst_av_audio_device_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstAVAudioDevice *device = GST_AV_AUDIO_DEVICE_CAST (object); + + switch (prop_id) { + case PROP_DEVICE_TYPE: + g_value_set_string (value, device->type); + break; + case PROP_DEVICE_UID: + g_value_set_string (value, device->uid); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_av_audio_device_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstAVAudioDevice *device = GST_AV_AUDIO_DEVICE_CAST (object); + + switch (prop_id) { + case PROP_DEVICE_TYPE: + device->type = g_strdup (g_value_get_string (value)); + break; + case PROP_DEVICE_UID: + device->uid = g_strdup (g_value_get_string (value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_av_audio_device_finalize (GObject * object) +{ + GstAVAudioDevice *device = GST_AV_AUDIO_DEVICE_CAST (object); + + g_free (device->type); + g_free (device->uid); + + G_OBJECT_CLASS (gst_av_audio_device_parent_class)->finalize (object); +} + +static void +gst_av_audio_device_class_init (GstAVAudioDeviceClass * klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + + object_class->get_property = gst_av_audio_device_get_property; + object_class->set_property = gst_av_audio_device_set_property; + object_class->finalize = gst_av_audio_device_finalize; + + g_object_class_install_property (object_class, PROP_DEVICE_TYPE, + g_param_spec_string ("type", + "Type", + "The AVAudioSessionPort type for the device", + NULL, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (object_class, PROP_DEVICE_UID, + g_param_spec_string ("uid", + "UID", + "A system-assigned unique identifier (UID) for the device", + NULL, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS)); +} + +static void +gst_av_audio_device_init (GstAVAudioDevice * device) +{ +} diff --git a/subprojects/gst-plugins-bad/sys/applemedia/avfvideosrc.m b/subprojects/gst-plugins-bad/sys/applemedia/avfvideosrc.m index 19dbd68f3c8..1ceb4065a26 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/avfvideosrc.m +++ b/subprojects/gst-plugins-bad/sys/applemedia/avfvideosrc.m @@ -177,10 +177,6 @@ @interface GstAVFVideoSrcImpl : NSObject change_state (element, transition); + + if (transition == GST_STATE_CHANGE_READY_TO_NULL) + [self closeDevice]; + + return ret; +} + - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)aConnection @@ -1162,6 +1146,8 @@ static void gst_avf_video_src_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_avf_video_src_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); +static GstStateChangeReturn gst_avf_video_src_change_state ( + GstElement * element, GstStateChange transition); static GstCaps * gst_avf_video_src_get_caps (GstBaseSrc * basesrc, GstCaps * filter); static gboolean gst_avf_video_src_set_caps (GstBaseSrc * basesrc, @@ -1193,6 +1179,7 @@ static void gst_avf_video_src_set_context (GstElement * element, gobject_class->get_property = gst_avf_video_src_get_property; gobject_class->set_property = gst_avf_video_src_set_property; + gstelement_class->change_state = gst_avf_video_src_change_state; gstelement_class->set_context = gst_avf_video_src_set_context; gstbasesrc_class->get_caps = gst_avf_video_src_get_caps; @@ -1409,6 +1396,16 @@ static void gst_avf_video_src_set_context (GstElement * element, } } +static GstStateChangeReturn +gst_avf_video_src_change_state (GstElement * element, GstStateChange transition) +{ + GstStateChangeReturn ret; + + ret = [GST_AVF_VIDEO_SRC_IMPL (element) changeState: transition]; + + return ret; +} + static GstCaps * gst_avf_video_src_get_caps (GstBaseSrc * basesrc, GstCaps * filter) { diff --git a/subprojects/gst-plugins-bad/sys/applemedia/meson.build b/subprojects/gst-plugins-bad/sys/applemedia/meson.build index ebe839fb8ad..52f996c59a7 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/meson.build +++ b/subprojects/gst-plugins-bad/sys/applemedia/meson.build @@ -29,9 +29,9 @@ endif if ['darwin', 'ios'].contains(host_system) objc = meson.get_compiler('objc') - if not objc.has_argument('-fobjc-arc') - error('ARC is required for building') - endif +# if not objc.has_argument('-fobjc-arc') +# error('ARC is required for building') +# endif applemedia_objc_args += ['-fobjc-arc'] @@ -68,7 +68,8 @@ endif if host_system == 'ios' applemedia_sources += [ 'iosassetsrc.m', - 'iosglmemory.c' + 'iosglmemory.c', + 'avaudiodeviceprovider.m', ] applemedia_objc_args += ['-fobjc-abi-version=2', '-fobjc-legacy-dispatch'] diff --git a/subprojects/gst-plugins-bad/sys/applemedia/plugin.m b/subprojects/gst-plugins-bad/sys/applemedia/plugin.m index 9fc42d728a0..ec2b9910539 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/plugin.m +++ b/subprojects/gst-plugins-bad/sys/applemedia/plugin.m @@ -26,6 +26,7 @@ #ifdef HAVE_IOS #include "iosassetsrc.h" #include "iosglmemory.h" +#include "avaudiodeviceprovider.h" #endif #ifdef HAVE_AVFOUNDATION #include "avfvideosrc.h" @@ -52,7 +53,7 @@ static void enable_mt_mode (void) { - NSThread * th = [[NSThread alloc] init]; + NSThread *th =[[NSThread alloc] init]; [th start]; g_assert ([NSThread isMultiThreaded]); } @@ -70,6 +71,9 @@ res &= gst_element_register (plugin, "iosassetsrc", GST_RANK_SECONDARY, GST_TYPE_IOS_ASSET_SRC); + res &= + gst_device_provider_register (plugin, "avaudiodeviceprovider", GST_RANK_PRIMARY, + GST_TYPE_AV_AUDIO_DEVICE_PROVIDER); #else enable_mt_mode (); #endif @@ -82,10 +86,11 @@ res &= gst_element_register (plugin, "avsamplebufferlayersink", GST_RANK_NONE, GST_TYPE_AV_SAMPLE_VIDEO_SINK); res &= gst_device_provider_register (plugin, "avfdeviceprovider", - GST_RANK_PRIMARY, GST_TYPE_AVF_DEVICE_PROVIDER); + GST_RANK_PRIMARY, GST_TYPE_AVF_DEVICE_PROVIDER); #endif - res &= gst_element_register (plugin, "atdec", GST_RANK_MARGINAL, GST_TYPE_ATDEC); + res &= + gst_element_register (plugin, "atdec", GST_RANK_MARGINAL, GST_TYPE_ATDEC); #ifdef HAVE_VIDEOTOOLBOX /* Check if the framework actually exists at runtime */ diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c index 4c2fb147b5f..f8a1cdf0f9f 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c +++ b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c @@ -74,8 +74,8 @@ #include #define VTENC_DEFAULT_BITRATE 0 -#define VTENC_DEFAULT_FRAME_REORDERING TRUE -#define VTENC_DEFAULT_REALTIME FALSE +#define VTENC_DEFAULT_FRAME_REORDERING FALSE +#define VTENC_DEFAULT_REALTIME TRUE #define VTENC_DEFAULT_QUALITY 0.5 #define VTENC_DEFAULT_MAX_KEYFRAME_INTERVAL 0 #define VTENC_DEFAULT_MAX_KEYFRAME_INTERVAL_DURATION 0 @@ -148,7 +148,8 @@ static void gst_vtenc_get_property (GObject * obj, guint prop_id, static void gst_vtenc_set_property (GObject * obj, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_vtenc_finalize (GObject * obj); - +static gboolean gst_vtenc_src_activate_mode (G_GNUC_UNUSED GstPad * pad, + GstObject * parent, GstPadMode mode, gboolean active); static gboolean gst_vtenc_start (GstVideoEncoder * enc); static gboolean gst_vtenc_stop (GstVideoEncoder * enc); static void gst_vtenc_loop (GstVTEnc * self); @@ -422,6 +423,7 @@ gst_vtenc_class_init (GstVTEncClass * klass) static void gst_vtenc_init (GstVTEnc * self) { + GstVideoEncoder *enc = GST_VIDEO_ENCODER_CAST (self); GstVTEncClass *klass = (GstVTEncClass *) G_OBJECT_GET_CLASS (self); CFStringRef keyframe_props_keys[] = { kVTEncodeFrameOptionKey_ForceKeyFrame }; CFBooleanRef keyframe_props_values[] = { kCFBooleanTrue }; @@ -443,6 +445,10 @@ gst_vtenc_init (GstVTEnc * self) g_mutex_init (&self->queue_mutex); g_cond_init (&self->queue_cond); + + gst_pad_set_activatemode_function (enc->srcpad, + GST_DEBUG_FUNCPTR (gst_vtenc_src_activate_mode)); + } static void @@ -698,6 +704,19 @@ gst_vtenc_pause_output_loop (GstVTEnc * self) g_mutex_unlock (&self->queue_mutex); } +static gboolean +gst_vtenc_src_activate_mode (G_GNUC_UNUSED GstPad * pad, + GstObject * parent, GstPadMode mode, gboolean active) +{ + GstVTEnc *self = GST_VTENC_CAST (parent); + GST_LOG_OBJECT (self, "activate mode %d active %d", mode, active); + + if (active == FALSE) + gst_vtenc_pause_output_loop (self); + + return TRUE; +} + static void gst_vtenc_set_flushing_flag (GstVTEnc * self) { @@ -830,6 +849,8 @@ gst_vtenc_stop (GstVideoEncoder * enc) return TRUE; } + + static gboolean gst_vtenc_h264_parse_profile_level_key (GstVTEnc * self, const gchar * profile, const gchar * level_arg) @@ -1004,6 +1025,8 @@ gst_vtenc_set_format (GstVideoEncoder * enc, GstVideoCodecState * state) gst_video_codec_state_unref (self->input_state); } + gst_vtenc_finish (enc); + GST_OBJECT_LOCK (self); gst_vtenc_destroy_session (self, &self->session); GST_OBJECT_UNLOCK (self); @@ -1030,6 +1053,13 @@ gst_vtenc_is_negotiated (GstVTEnc * self) return self->session && self->video_info.width != 0; } +static void +gst_vtenc_reset_session (GstVTEnc * self) +{ + gst_vtenc_destroy_session (self, &self->session); + self->session = gst_vtenc_create_session (self); +} + /* * When the image is opaque but the output ProRes format has an alpha * component (4 component, 32 bits per pixel), Apple requires that we signal @@ -1941,14 +1971,19 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame) GINT_TO_POINTER (frame->system_frame_number), NULL); GST_VIDEO_ENCODER_STREAM_LOCK (self); - if (vt_status != noErr) { - GST_WARNING_OBJECT (self, "VTCompressionSessionEncodeFrame returned %d", - (int) vt_status); - } - gst_video_codec_frame_unref (frame); CVPixelBufferRelease (pbuf); + if (vt_status == kVTInvalidSessionErr) { + GST_WARNING_OBJECT (self, "Invalid compression session, resetting."); + gst_vtenc_reset_session (self); + } else if (vt_status == kVTVideoEncoderMalfunctionErr) { + GST_WARNING_OBJECT (self, "Invalid compression session, resetting."); + gst_vtenc_reset_session (self); + } else if (vt_status != noErr) { + GST_WARNING_OBJECT (self, "VTCompressionSessionEncodeFrame returned %d", (int) vt_status); + } + return ret; drop: diff --git a/subprojects/gst-plugins-bad/sys/mediafoundation/gstmfsourcereader.cpp b/subprojects/gst-plugins-bad/sys/mediafoundation/gstmfsourcereader.cpp index f0066e3995f..844e328e41a 100644 --- a/subprojects/gst-plugins-bad/sys/mediafoundation/gstmfsourcereader.cpp +++ b/subprojects/gst-plugins-bad/sys/mediafoundation/gstmfsourcereader.cpp @@ -117,6 +117,9 @@ static gboolean gst_mf_source_enum_device_activate (GstMFSourceReader * self, GstMFSourceType source_type, GList ** device_activates); static void gst_mf_device_activate_free (GstMFDeviceActivate * activate); +static GstFlowReturn +gst_mf_source_reader_read_sample (GstMFSourceReader * self); + #define gst_mf_source_reader_parent_class parent_class G_DEFINE_TYPE (GstMFSourceReader, gst_mf_source_reader, GST_TYPE_MF_SOURCE_OBJECT); @@ -283,6 +286,7 @@ compare_caps_func (gconstpointer a, gconstpointer b) static gboolean gst_mf_source_reader_open (GstMFSourceReader * self, IMFActivate * activate) { + GstMFSourceObject *object = GST_MF_SOURCE_OBJECT (self); GList *iter; HRESULT hr; ComPtr < IMFSourceReader > reader; @@ -332,6 +336,24 @@ gst_mf_source_reader_open (GstMFSourceReader * self, IMFActivate * activate) GST_DEBUG_OBJECT (self, "Available output caps %" GST_PTR_FORMAT, self->supported_caps); + /* if we have specified a device path, try to read a frame, this will return + an error if the device is busy. + Note: we only want to do this when a specific device is specified, as + this code is also used for camera enumeration */ + if (object->device_path != nullptr) { + if (!gst_mf_source_reader_set_caps (GST_MF_SOURCE_OBJECT (self), self->supported_caps)) { + gst_mf_source_reader_close (self); + return FALSE; + } + + if (gst_mf_source_reader_read_sample (self) != GST_FLOW_OK){ + gst_mf_source_reader_close (self); + return FALSE; + } + + gst_queue_array_clear (self->queue); + } + return TRUE; } diff --git a/subprojects/gst-plugins-bad/sys/wasapi/gstwasapiutil.c b/subprojects/gst-plugins-bad/sys/wasapi/gstwasapiutil.c index 8f935837eea..6683c82653b 100644 --- a/subprojects/gst-plugins-bad/sys/wasapi/gstwasapiutil.c +++ b/subprojects/gst-plugins-bad/sys/wasapi/gstwasapiutil.c @@ -385,6 +385,24 @@ gst_wasapi_util_get_devices (GstMMDeviceEnumerator * self, if (!enum_handle) return FALSE; + IMMDevice *default_render_device = NULL; + wchar_t * default_render_id = NULL; + hr = IMMDeviceEnumerator_GetDefaultAudioEndpoint (enum_handle, eRender, + eCommunications, &default_render_device); + if (hr == S_OK && default_render_device != NULL) { + hr = IMMDevice_GetId (default_render_device, &default_render_id); + IUnknown_Release(default_render_device); + } + + IMMDevice *default_capture_device = NULL; + wchar_t * default_capture_id = NULL; + hr = IMMDeviceEnumerator_GetDefaultAudioEndpoint (enum_handle, eCapture, + eCommunications, &default_capture_device); + if (hr == S_OK && default_capture_device != NULL) { + hr = IMMDevice_GetId (default_capture_device, &default_capture_id); + IUnknown_Release(default_capture_device); + } + hr = IMMDeviceEnumerator_EnumAudioEndpoints (enum_handle, eAll, dwStateMask, &device_collection); HR_FAILED_GOTO (hr, IMMDeviceEnumerator::EnumAudioEndpoints, err); @@ -433,6 +451,13 @@ gst_wasapi_util_get_devices (GstMMDeviceEnumerator * self, hr = IMMDevice_GetId (item, &wstrid); if (hr != S_OK) goto next; + + gboolean is_default = FALSE; + if (dataflow == eRender && default_render_id && wstrid && wcscmp(default_render_id, wstrid) == 0) + is_default = TRUE; + if (dataflow == eCapture && default_capture_id && wstrid && wcscmp(default_capture_id, wstrid) == 0) + is_default = TRUE; + strid = g_utf16_to_utf8 (wstrid, -1, NULL, NULL, NULL); CoTaskMemFree (wstrid); @@ -477,7 +502,8 @@ gst_wasapi_util_get_devices (GstMMDeviceEnumerator * self, props = gst_structure_new ("wasapi-proplist", "device.api", G_TYPE_STRING, "wasapi", "device.strid", G_TYPE_STRING, GST_STR_NULL (strid), - "wasapi.device.description", G_TYPE_STRING, description, NULL); + "wasapi.device.description", G_TYPE_STRING, description, + "is-default", G_TYPE_BOOLEAN, is_default, NULL); device = g_object_new (GST_TYPE_WASAPI_DEVICE, "device", strid, "display-name", description, "caps", caps, @@ -507,6 +533,10 @@ gst_wasapi_util_get_devices (GstMMDeviceEnumerator * self, res = TRUE; err: + if (default_render_id) + CoTaskMemFree (default_render_id); + if (default_capture_id) + CoTaskMemFree (default_capture_id); if (device_collection) IUnknown_Release (device_collection); return res; diff --git a/subprojects/gst-plugins-bad/sys/winks/ksvideohelpers.c b/subprojects/gst-plugins-bad/sys/winks/ksvideohelpers.c index a5b53c15bee..fee908176e4 100644 --- a/subprojects/gst-plugins-bad/sys/winks/ksvideohelpers.c +++ b/subprojects/gst-plugins-bad/sys/winks/ksvideohelpers.c @@ -547,7 +547,6 @@ ks_video_probe_filter_for_caps (HANDLE filter_handle) range->Specifier, &entry->is_rgb); if (media_structure == NULL) { - g_warning ("ks_video_format_to_structure returned NULL"); ks_video_media_type_free (entry); entry = NULL; } else if (src_vscc == NULL) { diff --git a/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.c b/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.c index a398bd643db..beadb6a4a5f 100644 --- a/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.c +++ b/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.c @@ -63,6 +63,7 @@ enum { PROP_0, PROP_MONITOR, + PROP_WINDOW, PROP_SHOW_CURSOR, PROP_X_POS, PROP_Y_POS, @@ -122,6 +123,11 @@ gst_gdiscreencapsrc_class_init (GstGDIScreenCapSrcClass * klass) "Monitor", "Which monitor to use (0 = 1st monitor and default)", 0, G_MAXINT, 0, G_PARAM_READWRITE)); + g_object_class_install_property (go_class, PROP_WINDOW, + g_param_spec_int ("window", "Window", + "The window handle (HWND) to capture.", 0, G_MAXINT, 0, + G_PARAM_READWRITE)); + g_object_class_install_property (go_class, PROP_SHOW_CURSOR, g_param_spec_boolean ("cursor", "Show mouse cursor", "Whether to show mouse cursor (default off)", @@ -169,6 +175,7 @@ gst_gdiscreencapsrc_init (GstGDIScreenCapSrc * src) src->capture_h = 0; src->monitor = 0; + src->window = 0; src->show_cursor = FALSE; gst_base_src_set_format (GST_BASE_SRC (src), GST_FORMAT_TIME); @@ -215,6 +222,9 @@ gst_gdiscreencapsrc_set_property (GObject * object, guint prop_id, } src->monitor = g_value_get_int (value); break; + case PROP_WINDOW: + src->window = g_value_get_int (value); + break; case PROP_SHOW_CURSOR: src->show_cursor = g_value_get_boolean (value); break; @@ -246,6 +256,9 @@ gst_gdiscreencapsrc_get_property (GObject * object, guint prop_id, case PROP_MONITOR: g_value_set_int (value, src->monitor); break; + case PROP_WINDOW: + g_value_set_int (value, src->window); + break; case PROP_SHOW_CURSOR: g_value_set_boolean (value, src->show_cursor); break; @@ -330,7 +343,7 @@ gst_gdiscreencapsrc_set_caps (GstBaseSrc * bsrc, GstCaps * caps) DeleteDC (src->memDC); /* Allocate */ - capture = GetDesktopWindow (); + capture = src->window > 0 ? (HWND) src->window : GetDesktopWindow (); device = GetDC (capture); src->hBitmap = CreateDIBSection (device, &(src->info), DIB_RGB_COLORS, (void **) &(src->dibMem), 0, 0); @@ -346,6 +359,17 @@ gst_gdiscreencapsrc_set_caps (GstBaseSrc * bsrc, GstCaps * caps) return TRUE; } +static RECT +gst_win32_get_window_rect (HWND wnd) +{ + RECT rect; + + if (!GetClientRect (wnd, &rect)) + ZeroMemory (&rect, sizeof (RECT)); + + return rect; +} + static GstCaps * gst_gdiscreencapsrc_get_caps (GstBaseSrc * bsrc, GstCaps * filter) { @@ -353,7 +377,10 @@ gst_gdiscreencapsrc_get_caps (GstBaseSrc * bsrc, GstCaps * filter) RECT rect_dst; GstCaps *caps; - src->screen_rect = rect_dst = gst_win32_get_monitor_rect (src->monitor); + if (src->window > 0) + src->screen_rect = rect_dst = gst_win32_get_window_rect ((HWND)src->window); + else + src->screen_rect = rect_dst = gst_win32_get_monitor_rect (src->monitor); if (src->capture_w && src->capture_h && src->capture_x + src->capture_w < rect_dst.right - rect_dst.left && @@ -539,7 +566,7 @@ gst_gdiscreencapsrc_screen_capture (GstGDIScreenCapSrc * src, GstBuffer * buf) height = -src->info.bmiHeader.biHeight; /* Capture screen */ - capture = GetDesktopWindow (); + capture = src->window > 0 ? (HWND) src->window : GetDesktopWindow (); winDC = GetWindowDC (capture); BitBlt (src->memDC, 0, 0, width, height, diff --git a/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.h b/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.h index 085e5328866..6b6516430fd 100644 --- a/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.h +++ b/subprojects/gst-plugins-bad/sys/winscreencap/gstgdiscreencapsrc.h @@ -53,6 +53,7 @@ struct _GstGDIScreenCapSrc gint capture_w; gint capture_h; gint monitor; + gint window; gboolean show_cursor; /* Source pad frame rate */ diff --git a/subprojects/gst-plugins-base/ext/gl/meson.build b/subprojects/gst-plugins-base/ext/gl/meson.build index ad514014e9a..a5b08fb7a65 100644 --- a/subprojects/gst-plugins-base/ext/gl/meson.build +++ b/subprojects/gst-plugins-base/ext/gl/meson.build @@ -124,9 +124,9 @@ if ['darwin', 'ios'].contains(host_system) endif objc = meson.get_compiler('objc') - if not objc.has_argument('-fobjc-arc') - error('ARC is required for building') - endif +# if not objc.has_argument('-fobjc-arc') +# error('ARC is required for building') +# endif gl_objc_args += ['-fobjc-arc'] endif diff --git a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c index acb6314ea07..fd5f256dd84 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.c @@ -141,12 +141,14 @@ enum { PROP_0, PROP_LATENCY, + PROP_REPORTED_MIN_LATENCY, PROP_TOLERANCE, PROP_PLC, PROP_MAX_ERRORS }; #define DEFAULT_LATENCY 0 +#define DEFAULT_REPORTED_MIN_LATENCY 0 #define DEFAULT_TOLERANCE 0 #define DEFAULT_PLC FALSE #define DEFAULT_DRAINABLE TRUE @@ -264,11 +266,19 @@ struct _GstAudioDecoderPrivate /* properties */ GstClockTime latency; + GstClockTime reported_min_latency; GstClockTime tolerance; gboolean plc; gboolean drainable; gboolean needs_format; + /* dtx */ + gboolean dtx_running; + GMutex dtx_lock; + GstClockID dtx_clock_id; + GstClockTime last_pts; + GstClockTime last_buffer_duration; + /* pending serialized sink events, will be sent from finish_frame() */ GList *pending_events; @@ -276,6 +286,10 @@ struct _GstAudioDecoderPrivate gboolean use_default_pad_acceptcaps; }; +#define GST_AUDIO_DECODER_DTX_GET_LOCK(dec) (&dec->priv->dtx_lock) +#define GST_AUDIO_DECODER_DTX_LOCK(dec) (g_mutex_lock (GST_AUDIO_DECODER_DTX_GET_LOCK (dec))) +#define GST_AUDIO_DECODER_DTX_UNLOCK(dec) (g_mutex_unlock (GST_AUDIO_DECODER_DTX_GET_LOCK (dec))) + /* cached quark to avoid contention on the global quark table lock */ #define META_TAG_AUDIO meta_tag_audio_quark static GQuark meta_tag_audio_quark; @@ -306,8 +320,11 @@ static GstFlowReturn gst_audio_decoder_chain (GstPad * pad, GstObject * parent, GstBuffer * buf); static gboolean gst_audio_decoder_src_query (GstPad * pad, GstObject * parent, GstQuery * query); +static gboolean gst_audio_decoder_src_activate_mode (G_GNUC_UNUSED GstPad * pad, + GstObject * parent, GstPadMode mode, gboolean active); static gboolean gst_audio_decoder_sink_query (GstPad * pad, GstObject * parent, GstQuery * query); +static void gst_audio_decoder_stop_dtx (GstAudioDecoder * dec); static void gst_audio_decoder_reset (GstAudioDecoder * dec, gboolean full); static gboolean gst_audio_decoder_decide_allocation_default (GstAudioDecoder * @@ -406,6 +423,13 @@ gst_audio_decoder_class_init (GstAudioDecoderClass * klass) 0, G_MAXINT64, DEFAULT_LATENCY, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_REPORTED_MIN_LATENCY, + g_param_spec_int64 ("reported-min-latency", + "The Reported Minimum Latency", + "The minimum latency reported to the latency query", 0, G_MAXINT64, + DEFAULT_REPORTED_MIN_LATENCY, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_TOLERANCE, g_param_spec_int64 ("tolerance", "Tolerance", "Perfect ts while timestamp jitter/imperfection within tolerance (ns)", @@ -485,6 +509,8 @@ gst_audio_decoder_init (GstAudioDecoder * dec, GstAudioDecoderClass * klass) GST_DEBUG_FUNCPTR (gst_audio_decoder_src_event)); gst_pad_set_query_function (dec->srcpad, GST_DEBUG_FUNCPTR (gst_audio_decoder_src_query)); + gst_pad_set_activatemode_function (dec->srcpad, + GST_DEBUG_FUNCPTR (gst_audio_decoder_src_activate_mode)); gst_element_add_pad (GST_ELEMENT (dec), dec->srcpad); GST_DEBUG_OBJECT (dec, "srcpad created"); @@ -493,9 +519,11 @@ gst_audio_decoder_init (GstAudioDecoder * dec, GstAudioDecoderClass * klass) g_queue_init (&dec->priv->frames); g_rec_mutex_init (&dec->stream_lock); + g_mutex_init (GST_AUDIO_DECODER_DTX_GET_LOCK (dec)); /* property default */ dec->priv->latency = DEFAULT_LATENCY; + dec->priv->reported_min_latency = DEFAULT_REPORTED_MIN_LATENCY; dec->priv->tolerance = DEFAULT_TOLERANCE; dec->priv->plc = DEFAULT_PLC; dec->priv->drainable = DEFAULT_DRAINABLE; @@ -596,6 +624,7 @@ gst_audio_decoder_finalize (GObject * object) } g_rec_mutex_clear (&dec->stream_lock); + g_mutex_clear (GST_AUDIO_DECODER_DTX_GET_LOCK (dec)); G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -1068,6 +1097,11 @@ gst_audio_decoder_output (GstAudioDecoder * dec, GstBuffer * buf) GST_TIME_ARGS (GST_BUFFER_DURATION (buf))); } + if (priv->ctx.min_latency != priv->reported_min_latency) { + gst_audio_decoder_set_latency (dec, + priv->reported_min_latency, priv->reported_min_latency); + } + again: inbuf = NULL; if (priv->agg && dec->priv->latency > 0 && @@ -1416,9 +1450,9 @@ gst_audio_decoder_finish_frame_or_subframe (GstAudioDecoder * dec, } } else if (G_UNLIKELY (frames > priv->frames.length)) { if (G_LIKELY (!priv->force)) { - GST_ELEMENT_WARNING (dec, STREAM, DECODE, - ("received more decoded frames %d than provided %d", frames, - priv->frames.length), (NULL)); + GST_DEBUG_OBJECT (dec, + "received more decoded frames %d than provided %d", frames, + priv->frames.length); } frames = priv->frames.length; } @@ -1515,6 +1549,9 @@ gst_audio_decoder_finish_frame_or_subframe (GstAudioDecoder * dec, GST_FRAMES_TO_CLOCK_TIME (samples, ctx->info.rate); } + priv->last_pts = GST_BUFFER_PTS (buf); + priv->last_buffer_duration = GST_BUFFER_DURATION (buf); + if (klass->transform_meta) { if (inbufs.length) { GList *l; @@ -1536,7 +1573,7 @@ gst_audio_decoder_finish_frame_or_subframe (GstAudioDecoder * dec, data.outbuf = buf; gst_buffer_foreach_meta (in_buf, foreach_metadata, &data); } else { - GST_WARNING_OBJECT (dec, + GST_DEBUG_OBJECT (dec, "Can't copy metadata because input buffers disappeared"); } } @@ -1641,6 +1678,122 @@ gst_audio_decoder_handle_frame (GstAudioDecoder * dec, return klass->handle_frame (dec, buffer); } +/* called with GST_AUDIO_DECODER_DTX_LOCK held */ +static void +gst_audio_decoder_dtx_wait_unlocked (GstAudioDecoder * dec, GstClockTime time) +{ + GstClock *clock = GST_ELEMENT_CLOCK (dec); + + if (!clock) + return; + + dec->priv->dtx_clock_id = gst_clock_new_single_shot_id (clock, time); + GST_LOG_OBJECT (dec, "Waiting for %" GST_TIME_FORMAT, GST_TIME_ARGS (time)); + + GST_AUDIO_DECODER_DTX_UNLOCK (dec); + gst_clock_id_wait (dec->priv->dtx_clock_id, NULL); + GST_AUDIO_DECODER_DTX_LOCK (dec); + + gst_clock_id_unref (dec->priv->dtx_clock_id); + dec->priv->dtx_clock_id = NULL; +} + +static void +gst_audio_decoder_dtx_task_func (gpointer user_data) +{ + GstAudioDecoder *dec = GST_AUDIO_DECODER_CAST (user_data); + GstAudioDecoderClass *klass = GST_AUDIO_DECODER_GET_CLASS (dec); + GstClockTime pts; + GstClockTime base_time; + + base_time = gst_element_get_base_time (GST_ELEMENT (dec)); + pts = dec->priv->last_pts; + + if (!GST_CLOCK_TIME_IS_VALID (pts)) { + GST_WARNING_OBJECT (dec, "Last pts is invalid, can't start DTX"); + gst_pad_pause_task (dec->srcpad); + return; + } + + GST_AUDIO_DECODER_DTX_LOCK (dec); + while (dec->priv->dtx_running) { + GstFlowReturn ret; + GstBuffer *buf; + + pts += dec->priv->last_buffer_duration; + + gst_audio_decoder_dtx_wait_unlocked (dec, base_time + pts); + + if (!dec->priv->dtx_running) + break; + + GST_DEBUG_OBJECT (dec, + "producing DTX packet with timestamp %" GST_TIME_FORMAT, + GST_TIME_ARGS (pts)); + + /* same is in handle_gap() we tell the subclass to conceal */ + buf = gst_buffer_new (); + GST_BUFFER_PTS (buf) = pts; + GST_BUFFER_DURATION (buf) = dec->priv->last_buffer_duration; + + GST_AUDIO_DECODER_DTX_UNLOCK (dec); + GST_AUDIO_DECODER_STREAM_LOCK (dec); + + ret = gst_audio_decoder_handle_frame (dec, klass, buf); + + GST_AUDIO_DECODER_STREAM_UNLOCK (dec); + GST_AUDIO_DECODER_DTX_LOCK (dec); + + if (ret != GST_FLOW_OK) + break; + } + GST_AUDIO_DECODER_DTX_UNLOCK (dec); + + gst_pad_pause_task (dec->srcpad); +} + +/* called with GST_AUDIO_DECODER_DTX_LOCK held */ +static void +gst_audio_decoder_start_dtx_unlocked (GstAudioDecoder * dec) +{ + if (dec->priv->dtx_running) { + GST_ERROR_OBJECT (dec, "Can't double start the DTX task!"); + g_assert_not_reached (); + return; + } + + GST_INFO_OBJECT (dec, "Starting DTX"); + dec->priv->dtx_running = TRUE; + + if (!gst_pad_start_task (dec->srcpad, gst_audio_decoder_dtx_task_func, dec, + NULL)) + g_assert_not_reached (); +} + +static void +gst_audio_decoder_stop_dtx (GstAudioDecoder * dec) +{ + GST_AUDIO_DECODER_DTX_LOCK (dec); + + if (!dec->priv->dtx_running) { + GST_AUDIO_DECODER_DTX_UNLOCK (dec); + return; + } + + GST_INFO_OBJECT (dec, "Stopping DTX"); + dec->priv->dtx_running = FALSE; + + if (dec->priv->dtx_clock_id) { + gst_clock_id_unschedule (dec->priv->dtx_clock_id); + g_assert (dec->priv->dtx_clock_id); + } + + GST_AUDIO_DECODER_DTX_UNLOCK (dec); + + if (!gst_pad_stop_task (dec->srcpad)) + g_assert_not_reached (); +} + /* maybe subclass configurable instead, but this allows for a whole lot of * raw samples, so at least quite some encoded ... */ #define GST_AUDIO_DECODER_MAX_SYNC 10 * 8 * 2 * 1024 @@ -2078,6 +2231,8 @@ gst_audio_decoder_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) GST_TIME_ARGS (GST_BUFFER_PTS (buffer)), GST_TIME_ARGS (GST_BUFFER_DURATION (buffer))); + gst_audio_decoder_stop_dtx (dec); + GST_AUDIO_DECODER_STREAM_LOCK (dec); if (G_UNLIKELY (dec->priv->ctx.input_caps == NULL && dec->priv->needs_format)) @@ -2258,6 +2413,8 @@ gst_audio_decoder_handle_gap (GstAudioDecoder * dec, GstEvent * event) GstClockTime timestamp, duration; gboolean needs_reconfigure = FALSE; + gst_audio_decoder_stop_dtx (dec); + /* Ensure we have caps first */ GST_AUDIO_DECODER_STREAM_LOCK (dec); if (!GST_AUDIO_INFO_IS_VALID (&dec->priv->ctx.info)) { @@ -2278,7 +2435,6 @@ gst_audio_decoder_handle_gap (GstAudioDecoder * dec, GstEvent * event) gst_pad_mark_reconfigure (dec->srcpad); } } - GST_AUDIO_DECODER_STREAM_UNLOCK (dec); gst_event_parse_gap (event, ×tamp, &duration); @@ -2294,12 +2450,16 @@ gst_audio_decoder_handle_gap (GstAudioDecoder * dec, GstEvent * event) GstAudioDecoderClass *klass = GST_AUDIO_DECODER_GET_CLASS (dec); GstBuffer *buf; - /* hand subclass empty frame with duration that needs covering */ - buf = gst_buffer_new (); - GST_BUFFER_PTS (buf) = timestamp; - GST_BUFFER_DURATION (buf) = duration; - /* best effort, not much error handling */ - gst_audio_decoder_handle_frame (dec, klass, buf); + /* asking to conceal 0 duration does not make sense, specially since + _finish_frame is picky about the buffer there actually having a size */ + if (duration > 0) { + /* hand subclass empty frame with duration that needs covering */ + buf = gst_buffer_new (); + GST_BUFFER_PTS (buf) = timestamp; + GST_BUFFER_DURATION (buf) = duration; + /* best effort, not much error handling */ + gst_audio_decoder_handle_frame (dec, klass, buf); + } ret = TRUE; dec->priv->expecting_discont_buf = TRUE; gst_event_unref (event); @@ -2317,6 +2477,9 @@ gst_audio_decoder_handle_gap (GstAudioDecoder * dec, GstEvent * event) gst_event_unref (event); } } + + GST_AUDIO_DECODER_STREAM_UNLOCK (dec); + return ret; } @@ -3043,6 +3206,7 @@ gst_audio_decoder_src_query_default (GstAudioDecoder * dec, GstQuery * query) max_latency = -1; else max_latency += dec->priv->ctx.max_latency; + GST_OBJECT_UNLOCK (dec); gst_query_set_latency (query, live, min_latency, max_latency); @@ -3057,6 +3221,21 @@ gst_audio_decoder_src_query_default (GstAudioDecoder * dec, GstQuery * query) return res; } +static gboolean +gst_audio_decoder_src_activate_mode (G_GNUC_UNUSED GstPad * pad, + GstObject * parent, GstPadMode mode, gboolean active) +{ + GstAudioDecoder *dec; + + dec = GST_AUDIO_DECODER (parent); + GST_LOG_OBJECT (dec, "activate mode %d active %d", mode, active); + + if (active == FALSE) + gst_audio_decoder_stop_dtx (dec); + + return TRUE; +} + static gboolean gst_audio_decoder_src_query (GstPad * pad, GstObject * parent, GstQuery * query) { @@ -3085,6 +3264,8 @@ gst_audio_decoder_stop (GstAudioDecoder * dec) klass = GST_AUDIO_DECODER_GET_CLASS (dec); + gst_audio_decoder_stop_dtx (dec); + if (klass->stop) { ret = klass->stop (dec); } @@ -3133,6 +3314,9 @@ gst_audio_decoder_get_property (GObject * object, guint prop_id, case PROP_LATENCY: g_value_set_int64 (value, dec->priv->latency); break; + case PROP_REPORTED_MIN_LATENCY: + g_value_set_int64 (value, dec->priv->ctx.min_latency); + break; case PROP_TOLERANCE: g_value_set_int64 (value, dec->priv->tolerance); break; @@ -3160,6 +3344,9 @@ gst_audio_decoder_set_property (GObject * object, guint prop_id, case PROP_LATENCY: dec->priv->latency = g_value_get_int64 (value); break; + case PROP_REPORTED_MIN_LATENCY: + dec->priv->reported_min_latency = g_value_get_int64 (value); + break; case PROP_TOLERANCE: dec->priv->tolerance = g_value_get_int64 (value); break; @@ -3428,9 +3615,12 @@ gst_audio_decoder_set_latency (GstAudioDecoder * dec, GST_OBJECT_UNLOCK (dec); /* post latency message on the bus */ - if (post_message) + if (post_message) { gst_element_post_message (GST_ELEMENT (dec), gst_message_new_latency (GST_OBJECT (dec))); + + gst_audio_decoder_push_event (dec, gst_event_new_latency_changed ()); + } } /** @@ -3725,6 +3915,39 @@ gst_audio_decoder_get_needs_format (GstAudioDecoder * dec) return result; } +/** + * gst_audio_decoder_start_dtx: + * @dec: a #GstAudioDecoder + * + * Starts the DTX task. The audio decoder will spawn a task that will generate + * the silent buffers to cover the period of time we haven't received frames for. + * + * MT safe. + */ +void +gst_audio_decoder_start_dtx (GstAudioDecoder * dec) +{ + g_return_if_fail (GST_IS_AUDIO_DECODER (dec)); + + GST_AUDIO_DECODER_DTX_LOCK (dec); + gst_audio_decoder_start_dtx_unlocked (dec); + GST_AUDIO_DECODER_DTX_UNLOCK (dec); +} + +/** + * gst_audio_decoder_dtx_running: + * @dec: a #GstAudioDecoder + * + * Returns: TRUE if DTX has started on this decoder. + * + * MT safe. + */ +gboolean +gst_audio_decoder_dtx_running (GstAudioDecoder * dec) +{ + return dec->priv->dtx_running; +} + /** * gst_audio_decoder_merge_tags: * @dec: a #GstAudioDecoder diff --git a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.h b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.h index 03fb675390e..d7fb323fd42 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudiodecoder.h @@ -434,6 +434,12 @@ void gst_audio_decoder_set_needs_format (GstAudioDecoder * dec, GST_AUDIO_API gboolean gst_audio_decoder_get_needs_format (GstAudioDecoder * dec); +GST_AUDIO_API +void gst_audio_decoder_start_dtx (GstAudioDecoder * dec); + +GST_AUDIO_API +gboolean gst_audio_decoder_dtx_running (GstAudioDecoder * dec); + GST_AUDIO_API void gst_audio_decoder_get_allocator (GstAudioDecoder * dec, GstAllocator ** allocator, diff --git a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudioencoder.c b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudioencoder.c index f77216709bd..0ccf0bde64e 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudioencoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/audio/gstaudioencoder.c @@ -918,6 +918,16 @@ gst_audio_encoder_finish_frame (GstAudioEncoder * enc, GstBuffer * buf, /* collect output */ if (G_LIKELY (buf)) { gsize size; + GstClockTime timestamp = GST_CLOCK_TIME_NONE; + + if (G_LIKELY (GST_CLOCK_TIME_IS_VALID (priv->base_ts))) { + /* FIXME ? lookahead could lead to weird ts and duration ? + * (particularly if not in perfect mode) */ + /* mind sample rounding and produce perfect output */ + timestamp = priv->base_ts + + gst_util_uint64_scale (priv->samples - ctx->lookahead, GST_SECOND, + ctx->info.rate); + } /* Pushing headers first */ if (G_UNLIKELY (priv->ctx.new_headers)) { @@ -931,6 +941,12 @@ gst_audio_encoder_finish_frame (GstAudioEncoder * enc, GstBuffer * buf, tmpbuf = gst_buffer_make_writable (tmpbuf); size = gst_buffer_get_size (tmpbuf); + /* we timestamp the headers with the same timestamp as the first + buffer, and set their duration to 0 */ + GST_BUFFER_PTS (tmpbuf) = timestamp; + GST_BUFFER_DTS (tmpbuf) = timestamp; + GST_BUFFER_DURATION (tmpbuf) = 0; + if (G_UNLIKELY (priv->discont)) { GST_LOG_OBJECT (enc, "marking discont"); GST_BUFFER_FLAG_SET (tmpbuf, GST_BUFFER_FLAG_DISCONT); @@ -971,10 +987,8 @@ gst_audio_encoder_finish_frame (GstAudioEncoder * enc, GstBuffer * buf, /* FIXME ? lookahead could lead to weird ts and duration ? * (particularly if not in perfect mode) */ /* mind sample rounding and produce perfect output */ - GST_BUFFER_PTS (buf) = priv->base_ts + - gst_util_uint64_scale (priv->samples - ctx->lookahead, GST_SECOND, - ctx->info.rate); - GST_BUFFER_DTS (buf) = GST_BUFFER_PTS (buf); + GST_BUFFER_PTS (buf) = timestamp; + GST_BUFFER_DTS (buf) = timestamp; GST_DEBUG_OBJECT (enc, "out samples %d", samples); if (G_LIKELY (samples > 0)) { priv->samples += samples; diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c index 07c6386954c..3d2206bde9a 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c @@ -1432,7 +1432,7 @@ gst_gl_color_convert_fixate_format_target (GstCaps * caps, GstCaps * result) const GstVideoFormatInfo *in_info, *out_info = NULL; const GValue *targets; guint targets_mask = 0; - GstGLTextureTarget target; + GstGLTextureTarget target = GST_GL_TEXTURE_TARGET_NONE; gint min_loss = G_MAXINT; guint i, capslen; @@ -1496,7 +1496,7 @@ gst_gl_color_convert_fixate_format_target (GstCaps * caps, GstCaps * result) if (out_info) gst_structure_set (outs, "format", G_TYPE_STRING, GST_VIDEO_FORMAT_INFO_NAME (out_info), NULL); - if (target) + if (target != GST_GL_TEXTURE_TARGET_NONE) gst_structure_set (outs, "texture-target", G_TYPE_STRING, gst_gl_texture_target_to_string (target), NULL); } @@ -2713,7 +2713,7 @@ _init_convert (GstGLColorConvert * convert) info->cms_coeff3); } - for (i = info->in_n_textures; i >= 0; i--) { + for (i = info->in_n_textures - 1; i >= 0; i--) { if (info->shader_tex_names[i]) gst_gl_shader_set_uniform_1i (convert->shader, info->shader_tex_names[i], i); diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglshader.c b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglshader.c index 09bdda4321a..05509fab4ae 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglshader.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglshader.c @@ -113,7 +113,7 @@ _cleanup_shader (GstGLContext * context, GstGLShader * shader) } static void -gst_gl_shader_finalize (GObject * object) +gst_gl_shader_dispose (GObject * object) { GstGLShader *shader; GstGLShaderPrivate *priv; @@ -134,7 +134,7 @@ gst_gl_shader_finalize (GObject * object) shader->context = NULL; } - G_OBJECT_CLASS (gst_gl_shader_parent_class)->finalize (object); + G_OBJECT_CLASS (gst_gl_shader_parent_class)->dispose (object); } static void @@ -171,7 +171,7 @@ gst_gl_shader_class_init (GstGLShaderClass * klass) /* bind class methods .. */ GObjectClass *obj_class = G_OBJECT_CLASS (klass); - obj_class->finalize = gst_gl_shader_finalize; + obj_class->dispose = gst_gl_shader_dispose; obj_class->set_property = gst_gl_shader_set_property; obj_class->get_property = gst_gl_shader_get_property; @@ -755,7 +755,9 @@ gst_gl_shader_release_unlocked (GstGLShader * shader) priv->linked = FALSE; g_hash_table_remove_all (priv->uniform_locations); + GST_OBJECT_UNLOCK (shader); g_object_notify (G_OBJECT (shader), "linked"); + GST_OBJECT_LOCK (shader); } /** diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglupload.c b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglupload.c index 0124ba89123..90efb24c05f 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglupload.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglupload.c @@ -101,6 +101,8 @@ struct _GstGLUploadPrivate GstCaps *in_caps; GstCaps *out_caps; + GstVideoAlignment align; + GstBuffer *outbuf; /* all method impl pointers */ @@ -2076,7 +2078,9 @@ static struct RawUploadFrame * _raw_upload_frame_new (struct RawUpload *raw, GstBuffer * buffer) { struct RawUploadFrame *frame; + GstVideoFrame *vframe; GstVideoInfo *info; + GstVideoMeta *vmeta; gint i; if (!buffer) @@ -2084,13 +2088,17 @@ _raw_upload_frame_new (struct RawUpload *raw, GstBuffer * buffer) frame = g_new (struct RawUploadFrame, 1); frame->ref_count = 1; + vframe = &frame->frame; - if (!gst_video_frame_map (&frame->frame, &raw->upload->priv->in_info, + if (!gst_video_frame_map (vframe, &raw->upload->priv->in_info, buffer, GST_MAP_READ)) { g_free (frame); return NULL; } + vmeta = vframe->meta; + if (vmeta) + raw->upload->priv->align = vmeta->alignment; raw->upload->priv->in_info = frame->frame.info; info = &raw->upload->priv->in_info; @@ -2210,7 +2218,7 @@ _raw_data_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps, gst_gl_allocation_params_free ((GstGLAllocationParams *) raw->params); if (!(raw->params = gst_gl_video_allocation_params_new_wrapped_data (raw->upload->context, - NULL, &raw->upload->priv->in_info, -1, NULL, + NULL, &raw->upload->priv->in_info, -1, &raw->upload->priv->align, GST_GL_TEXTURE_TARGET_2D, 0, NULL, raw->in_frame, (GDestroyNotify) _raw_upload_frame_unref))) return FALSE; @@ -3167,6 +3175,7 @@ static void gst_gl_upload_init (GstGLUpload * upload) { upload->priv = gst_gl_upload_get_instance_private (upload); + gst_video_alignment_reset (&upload->priv->align); } /** diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/meson.build b/subprojects/gst-plugins-base/gst-libs/gst/gl/meson.build index ebdab9089ab..ad3bd339d33 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/gl/meson.build +++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/meson.build @@ -800,9 +800,9 @@ if ['darwin', 'ios'].contains(host_system) endif objc = meson.get_compiler('objc') - if not objc.has_argument('-fobjc-arc') - error('ARC is required for building') - endif + #if not objc.has_argument('-fobjc-arc') + # error('ARC is required for building') + #endif gl_objc_args += ['-fobjc-arc'] endif diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.c b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.c index dea2a7933d5..49664b679c5 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.c @@ -91,12 +91,9 @@ static gboolean gst_rtcp_buffer_validate_data_internal (guint8 * data, guint len, guint16 valid_mask) { - guint16 header_mask; - guint header_len; guint8 version; + guint16 header_mask; guint data_len; - gboolean padding; - guint8 pad_bytes; g_return_val_if_fail (data != NULL, FALSE); @@ -109,43 +106,49 @@ gst_rtcp_buffer_validate_data_internal (guint8 * data, guint len, if (G_UNLIKELY (header_mask != GST_RTCP_VALID_VALUE)) goto wrong_mask; - padding = data[0] & 0x20; - /* store len */ data_len = len; while (TRUE) { + guint header_len; + gboolean padding = FALSE; + /* get packet length */ header_len = (((data[2] << 8) | data[3]) + 1) << 2; if (data_len < header_len) goto wrong_length; + /* check padding of new packet */ + if (data[0] & 0x20) { + guint8 pad_bytes; + padding = TRUE; + /* last byte of padding contains the number of padded bytes including + * itself */ + pad_bytes = data[data_len - 1]; + /* cannot be 0 and must be smaller than the size of the data */ + if (pad_bytes == 0 || pad_bytes > (header_len - 4)) + goto wrong_padding; + /* if validating a standard RTCP packet, padding must be modulo 4 */ + if (GST_RTCP_VALID_MASK == valid_mask && (pad_bytes & 0x3)) + goto wrong_padding; + } + /* move to next compound packet */ data += header_len; data_len -= header_len; - /* we are at the end now */ - if (data_len < 4) + /* padding only allowed on last packet */ + if (padding) break; - /* Version already checked for first packet through mask */ - if (padding) + /* we are at the end now */ + if (data_len < 4) break; /* check version of new packet */ version = data[0] & 0xc0; if (version != (GST_RTCP_VERSION << 6)) goto wrong_version; - - /* check padding of new packet */ - if (data[0] & 0x20) { - padding = TRUE; - /* last byte of padding contains the number of padded bytes including - * itself. must be a multiple of 4, but cannot be 0. */ - pad_bytes = data[data_len - 1]; - if (pad_bytes == 0 || (pad_bytes & 0x3)) - goto wrong_padding; - } } if (data_len != 0) { /* some leftover bytes */ @@ -2324,6 +2327,36 @@ gst_rtcp_packet_fb_get_fci_length (GstRTCPPacket * packet) return GST_READ_UINT16_BE (data) - 2; } +/** + * gst_rtcp_packet_fb_get_fci_length_bytes: + * @packet: a valid RTPFB or PSFB #GstRTCPPacket + * + * Get the length of the Feedback Control Information attached to a + * RTPFB or PSFB @packet. + * + * Returns: The length of the FCI in bytes. + */ +guint16 +gst_rtcp_packet_fb_get_fci_length_bytes (GstRTCPPacket * packet) +{ + guint16 wordlen; + guint16 byteslen; + + wordlen = gst_rtcp_packet_fb_get_fci_length (packet); + byteslen = wordlen * 4; + + /* subtract any padding */ + if (packet->padding) { + guint8 *data; + guint8 pad_bytes; + data = packet->rtcp->map.data + packet->offset; + pad_bytes = data[4 + packet->length * 4 - 1]; + byteslen -= pad_bytes; + } + + return byteslen; +} + /** * gst_rtcp_packet_fb_set_fci_length: * @packet: a valid RTPFB or PSFB #GstRTCPPacket diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.h b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.h index b6410a5a1f5..4173d288b2c 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtcpbuffer.h @@ -255,9 +255,9 @@ typedef enum /** * GST_RTCP_VALID_MASK: * - * Mask for version, padding bit and packet type pair + * Mask for version and packet type pair */ -#define GST_RTCP_VALID_MASK (0xc000 | 0x2000 | 0xfe) +#define GST_RTCP_VALID_MASK (0xc000 | 0xfe) /** * GST_RTCP_REDUCED_SIZE_VALID_MASK: @@ -546,6 +546,9 @@ void gst_rtcp_packet_fb_set_type (GstRTCPPacket *packet, Gs GST_RTP_API guint16 gst_rtcp_packet_fb_get_fci_length (GstRTCPPacket *packet); +GST_RTP_API +guint16 gst_rtcp_packet_fb_get_fci_length_bytes (GstRTCPPacket * packet); + GST_RTP_API gboolean gst_rtcp_packet_fb_set_fci_length (GstRTCPPacket *packet, guint16 wordlen); diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c index c788898ef28..4aadd592e56 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasedepayload.c @@ -84,13 +84,15 @@ struct _GstRTPBaseDepayloadPrivate GstClockTime pts; GstClockTime dts; GstClockTime duration; + guint64 offset; GstClockTime ref_ts; - + guint64 first_rtptime; guint32 last_ssrc; guint32 last_seqnum; guint32 last_rtptime; guint32 next_seqnum; + gboolean ignore_gaps; gint max_reorder; gboolean auto_hdr_ext; @@ -128,6 +130,7 @@ enum static guint gst_rtp_base_depayload_signals[LAST_SIGNAL] = { 0 }; +#define DEFAULT_IGNORE_GAPS FALSE #define DEFAULT_SOURCE_INFO FALSE #define DEFAULT_MAX_REORDER 100 #define DEFAULT_AUTO_HEADER_EXTENSION TRUE @@ -137,6 +140,7 @@ enum PROP_0, PROP_STATS, PROP_SOURCE_INFO, + PROP_IGNORE_GAPS, PROP_MAX_REORDER, PROP_AUTO_HEADER_EXTENSION, PROP_LAST @@ -345,6 +349,19 @@ gst_rtp_base_depayload_class_init (GstRTPBaseDepayloadClass * klass) DEFAULT_AUTO_HEADER_EXTENSION, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + /** + * GstRTPBaseDepayload:ignore-gaps: + * + * Wether the depayoader should looks at sequence-number gaps, + * and set a DISCONT flag accordingly. + * + * Since: 1.20 + **/ + g_object_class_install_property (gobject_class, PROP_IGNORE_GAPS, + g_param_spec_boolean ("ignore-gaps", "Ignore Gaps", + "Wether the depayloader should ignore sequencenumber gaps", + DEFAULT_IGNORE_GAPS, G_PARAM_READWRITE)); + /** * GstRTPBaseDepayload::request-extension: * @object: the #GstRTPBaseDepayload @@ -404,6 +421,8 @@ gst_rtp_base_depayload_class_init (GstRTPBaseDepayloadClass * klass) "Base class for RTP Depayloaders"); } +#define RTPTIME_NONE G_MAXUINT64 + static void gst_rtp_base_depayload_init (GstRTPBaseDepayload * filter, GstRTPBaseDepayloadClass * klass) @@ -445,7 +464,10 @@ gst_rtp_base_depayload_init (GstRTPBaseDepayload * filter, priv->pts = -1; priv->duration = -1; priv->ref_ts = -1; + priv->first_rtptime = RTPTIME_NONE; + priv->offset = GST_BUFFER_OFFSET_NONE; priv->source_info = DEFAULT_SOURCE_INFO; + priv->ignore_gaps = DEFAULT_IGNORE_GAPS; priv->max_reorder = DEFAULT_MAX_REORDER; priv->auto_hdr_ext = DEFAULT_AUTO_HEADER_EXTENSION; priv->hdrext_aggregate = FALSE; @@ -772,6 +794,12 @@ gst_rtp_base_depayload_handle_buffer (GstRTPBaseDepayload * filter, seqnum = gst_rtp_buffer_get_seq (&rtp); rtptime = gst_rtp_buffer_get_timestamp (&rtp); + if (priv->first_rtptime == RTPTIME_NONE) { + // First buffer handled + priv->first_rtptime = rtptime; + } + priv->offset = rtptime - priv->first_rtptime; + priv->last_seqnum = seqnum; priv->last_rtptime = rtptime; @@ -794,7 +822,7 @@ gst_rtp_base_depayload_handle_buffer (GstRTPBaseDepayload * filter, gap = gst_rtp_buffer_compare_seqnum (seqnum, priv->next_seqnum); /* if we have no gap, all is fine */ - if (G_UNLIKELY (gap != 0)) { + if (G_UNLIKELY (gap != 0) && !priv->ignore_gaps) { GST_LOG_OBJECT (filter, "got packet %u, expected %u, gap %d", seqnum, priv->next_seqnum, gap); if (gap < 0) { @@ -1398,6 +1426,9 @@ gst_rtp_base_depayload_set_headers (GstRTPBaseDepayload * depayload, GST_BUFFER_DTS (buffer) = priv->dts; if (!GST_CLOCK_TIME_IS_VALID (duration)) GST_BUFFER_DURATION (buffer) = priv->duration; + if (!GST_BUFFER_OFFSET_IS_VALID (buffer)) { + GST_BUFFER_OFFSET (buffer) = priv->offset; + } if (G_UNLIKELY (depayload->priv->discont)) { GST_LOG_OBJECT (depayload, "Marking DISCONT on output buffer"); @@ -1409,6 +1440,7 @@ gst_rtp_base_depayload_set_headers (GstRTPBaseDepayload * depayload, priv->pts = GST_CLOCK_TIME_NONE; priv->dts = GST_CLOCK_TIME_NONE; priv->duration = GST_CLOCK_TIME_NONE; + priv->offset = GST_BUFFER_OFFSET_NONE; if (priv->input_buffer) { if (priv->source_info) @@ -1624,6 +1656,7 @@ gst_rtp_base_depayload_packet_lost (GstRTPBaseDepayload * filter, const GstStructure *s; gboolean might_have_been_fec; gboolean res = TRUE; + gboolean noloss = FALSE; s = gst_event_get_structure (event); @@ -1637,6 +1670,7 @@ gst_rtp_base_depayload_packet_lost (GstRTPBaseDepayload * filter, "Packet loss event without timestamp or duration"); return FALSE; } + gst_structure_get_boolean (s, "no-packet-loss", &noloss); sevent = gst_pad_get_sticky_event (filter->srcpad, GST_EVENT_SEGMENT, 0); if (G_UNLIKELY (!sevent)) { @@ -1651,7 +1685,8 @@ gst_rtp_base_depayload_packet_lost (GstRTPBaseDepayload * filter, &might_have_been_fec) || !might_have_been_fec) { /* send GAP event */ sevent = gst_event_new_gap (timestamp, duration); - gst_event_set_gap_flags (sevent, GST_GAP_FLAG_MISSING_DATA); + if (!noloss) + gst_event_set_gap_flags (sevent, GST_GAP_FLAG_MISSING_DATA); res = gst_pad_push_event (filter->srcpad, sevent); } @@ -1685,6 +1720,7 @@ gst_rtp_base_depayload_change_state (GstElement * element, priv->negotiated = FALSE; priv->discont = FALSE; priv->segment_seqnum = GST_SEQNUM_INVALID; + priv->first_rtptime = RTPTIME_NONE; break; case GST_STATE_CHANGE_PAUSED_TO_PLAYING: break; @@ -1757,6 +1793,9 @@ gst_rtp_base_depayload_set_property (GObject * object, guint prop_id, gst_rtp_base_depayload_set_source_info_enabled (depayload, g_value_get_boolean (value)); break; + case PROP_IGNORE_GAPS: + priv->ignore_gaps = g_value_get_boolean (value); + break; case PROP_MAX_REORDER: priv->max_reorder = g_value_get_int (value); break; @@ -1788,6 +1827,9 @@ gst_rtp_base_depayload_get_property (GObject * object, guint prop_id, g_value_set_boolean (value, gst_rtp_base_depayload_is_source_info_enabled (depayload)); break; + case PROP_IGNORE_GAPS: + g_value_set_boolean (value, priv->ignore_gaps); + break; case PROP_MAX_REORDER: g_value_set_int (value, priv->max_reorder); break; diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasepayload.c b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasepayload.c index 02d0f51d234..17261827e4d 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasepayload.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbasepayload.c @@ -105,7 +105,7 @@ static guint gst_rtp_base_payload_signals[LAST_SIGNAL] = { 0 }; #define DEFAULT_SEQNUM_OFFSET -1 #define DEFAULT_MAX_PTIME -1 #define DEFAULT_MIN_PTIME 0 -#define DEFAULT_PERFECT_RTPTIME TRUE +#define DEFAULT_PERFECT_RTPTIME FALSE #define DEFAULT_PTIME_MULTIPLE 0 #define DEFAULT_RUNNING_TIME GST_CLOCK_TIME_NONE #define DEFAULT_SOURCE_INFO FALSE diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.c b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.c index ec4b53bd7be..466558dc0b5 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.c @@ -34,6 +34,7 @@ #endif #include "gstrtpbuffer.h" +#include #include #include @@ -1766,3 +1767,88 @@ gst_rtp_buffer_add_extension_twobytes_header (GstRTPBuffer * rtp, return TRUE; } + +/** + * gst_rtp_buffer_video_roi_meta_to_one_byte_ext: + * @rtp: a #GstRTPBuffer to write a ROI header extension to. + * @buffer: a #GstBuffer to extract #GstVideoRegionOfInterestMeta from. + * @id: The ID of the header extension to be added (between 1 and 14). + * + * Tries to read #GstVideoRegionOfInterestMeta from to the @buffer, and + * add this as ROI information in @rtp using the one byte + * extension header. + * + * Returns: %TRUE if any extension-headers was added, %FALSE otherwise. + * + * Since: 1.18 + */ +gboolean +gst_rtp_buffer_video_roi_meta_to_one_byte_ext (GstRTPBuffer * rtp, + GstBuffer * buffer, guint8 id) +{ + GstVideoRegionOfInterestMeta *meta; + gpointer state = NULL; + gboolean ret = FALSE; + + g_return_val_if_fail (buffer != NULL, FALSE); + g_return_val_if_fail (rtp != NULL, FALSE); + g_return_val_if_fail (id > 0, FALSE); + g_return_val_if_fail (id <= 14, FALSE); + + while ((meta = (GstVideoRegionOfInterestMeta *) + gst_buffer_iterate_meta_filtered (buffer, &state, + GST_VIDEO_REGION_OF_INTEREST_META_API_TYPE))) { + guint8 data[9]; + GST_WRITE_UINT16_BE (&data[0], meta->x); + GST_WRITE_UINT16_BE (&data[2], meta->y); + GST_WRITE_UINT16_BE (&data[4], meta->w); + GST_WRITE_UINT16_BE (&data[6], meta->h); + GST_WRITE_UINT8 (&data[8], meta->roi_type); + gst_rtp_buffer_add_extension_onebyte_header (rtp, id, &data, 9); + ret = TRUE; + } + + return ret; +} + +/** + * gst_rtp_buffer_video_roi_meta_from_one_byte_ext: + * @rtp: a #GstRTPBuffer to read a ROI header extension to. + * @buffer: a #GstBuffer to write #GstVideoRegionOfInterestMeta to. + * @id: The ID of the header extension to be read (between 1 and 14). + * + * Tries to extract ROI information from the RTP packet + * and adds this as #GstVideoRegionOfInterestMeta on to the @buffer. + * + * Returns: %TRUE if any meta was extracted, %FALSE otherwise. + * + * Since: 1.18 + */ +gboolean +gst_rtp_buffer_video_roi_meta_from_one_byte_ext (GstRTPBuffer * rtp, + GstBuffer * buffer, guint8 id) +{ + gpointer data; + guint nth = 0; + + g_return_val_if_fail (buffer != NULL, FALSE); + g_return_val_if_fail (rtp != NULL, FALSE); + g_return_val_if_fail (id > 0, FALSE); + g_return_val_if_fail (id <= 14, FALSE); + + while (gst_rtp_buffer_get_extension_onebyte_header (rtp, id, nth, &data, + NULL)) { + const guint8 *bytes = (const guint8 *) data; + guint16 x = GST_READ_UINT16_BE (&bytes[0]); + guint16 y = GST_READ_UINT16_BE (&bytes[2]); + guint16 w = GST_READ_UINT16_BE (&bytes[4]); + guint16 h = GST_READ_UINT16_BE (&bytes[6]); + guint16 roi_type = GST_READ_UINT8 (&bytes[8]); + + gst_buffer_add_video_region_of_interest_meta_id (buffer, roi_type, x, y, w, + h); + nth++; + } + + return nth > 0; +} diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h index cac8998c2b8..8b08b57d292 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtpbuffer.h @@ -239,6 +239,16 @@ gboolean gst_rtp_buffer_get_extension_onebyte_header_from_bytes (GBytes * bytes, gpointer * data, guint * size); +GST_RTP_API +gboolean gst_rtp_buffer_video_roi_meta_to_one_byte_ext (GstRTPBuffer * rtp, + GstBuffer * buf, + guint8 id); + +GST_RTP_API +gboolean gst_rtp_buffer_video_roi_meta_from_one_byte_ext (GstRTPBuffer * rtp, + GstBuffer * buffer, + guint8 id); + /** * GstRTPBufferFlags: * @GST_RTP_BUFFER_FLAG_RETRANSMISSION: The #GstBuffer was once wrapped @@ -260,6 +270,9 @@ gboolean gst_rtp_buffer_get_extension_onebyte_header_from_bytes (GBytes * bytes, typedef enum { GST_RTP_BUFFER_FLAG_RETRANSMISSION = (GST_BUFFER_FLAG_LAST << 0), GST_RTP_BUFFER_FLAG_REDUNDANT = (GST_BUFFER_FLAG_LAST << 1), + GST_RTP_BUFFER_FLAG_MEDIA_AUDIO = (GST_BUFFER_FLAG_LAST << 2), + GST_RTP_BUFFER_FLAG_MEDIA_VIDEO = (GST_BUFFER_FLAG_LAST << 3), + GST_RTP_BUFFER_FLAG_ULPFEC = (GST_BUFFER_FLAG_LAST << 4), GST_RTP_BUFFER_FLAG_LAST = (GST_BUFFER_FLAG_LAST << 8) } GstRTPBufferFlags; diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtppayloads.h b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtppayloads.h index 84f6c3cc3cc..98d9f288701 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtppayloads.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/gstrtppayloads.h @@ -149,7 +149,7 @@ typedef enum #define GST_RTP_PAYLOAD_MP2T_STRING "33" #define GST_RTP_PAYLOAD_H263_STRING "34" -#define GST_RTP_PAYLOAD_DYNAMIC_STRING "[96, 127]" +#define GST_RTP_PAYLOAD_DYNAMIC_STRING "[0, 127]" /** * GST_RTP_PAYLOAD_IS_DYNAMIC: diff --git a/subprojects/gst-plugins-base/gst-libs/gst/rtp/meson.build b/subprojects/gst-plugins-base/gst-libs/gst/rtp/meson.build index 19a65ee7137..14bf1fdf1b8 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/rtp/meson.build +++ b/subprojects/gst-plugins-base/gst-libs/gst/rtp/meson.build @@ -34,7 +34,7 @@ rtp_enums = gnome.mkenums_simple('gstrtp-enumtypes', gstrtp_enum_c = rtp_enums[0] gstrtp_enum_h = rtp_enums[1] -gstrtp_deps = [audio_dep, gst_base_dep] +gstrtp_deps = [audio_dep, video_dep, gst_base_dep] gst_rtp = library('gstrtp-@0@'.format(api_version), rtp_sources, gstrtp_enum_c, gstrtp_enum_h, c_args : gst_plugins_base_args + ['-DBUILDING_GST_RTP', '-DG_LOG_DOMAIN="GStreamer-RTP"'], diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c index 74bcab1e728..777de8cc1e3 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c @@ -314,6 +314,7 @@ GST_DEBUG_CATEGORY (videodecoder_debug); * frame numbers and can be given special meaning */ #define REQUEST_SYNC_POINT_PENDING G_MAXUINT + 1 #define REQUEST_SYNC_POINT_UNSET G_MAXUINT64 +#define DEFAULT_DETECT_REORDERING TRUE enum { @@ -324,6 +325,8 @@ enum PROP_DISCARD_CORRUPTED_FRAMES, PROP_AUTOMATIC_REQUEST_SYNC_POINTS, PROP_AUTOMATIC_REQUEST_SYNC_POINT_FLAGS, + PROP_DETECT_REORDERING, + PROP_LAST }; struct _GstVideoDecoderPrivate @@ -386,6 +389,7 @@ struct _GstVideoDecoderPrivate /* incoming pts - dts */ GstClockTime pts_delta; gboolean reordered_output; + gboolean detect_reordering; /* FIXME: Consider using a GQueue or other better fitting data structure */ /* reverse playback */ @@ -716,6 +720,21 @@ gst_video_decoder_class_init (GstVideoDecoderClass * klass) DEFAULT_AUTOMATIC_REQUEST_SYNC_POINT_FLAGS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + /** + * GstVideoDecoder:detect-reordering: + * + * If set to %FALSE the decoder will not bother with reordered frames, and + * do any sort of PTS adjustment for them. + * + * Since: 1.20 + */ + g_object_class_install_property (gobject_class, PROP_DETECT_REORDERING, + g_param_spec_boolean ("detect-reordering", + "Detect frame reordering", + "Enables the frame pts reordering detection", + DEFAULT_DETECT_REORDERING, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + meta_tag_video_quark = g_quark_from_static_string (GST_META_TAG_VIDEO_STR); } @@ -770,6 +789,7 @@ gst_video_decoder_init (GstVideoDecoder * decoder, GstVideoDecoderClass * klass) /* properties */ decoder->priv->do_qos = DEFAULT_QOS; decoder->priv->max_errors = GST_VIDEO_DECODER_MAX_ERRORS; + decoder->priv->detect_reordering = DEFAULT_DETECT_REORDERING; decoder->priv->min_latency = 0; decoder->priv->max_latency = 0; @@ -995,6 +1015,9 @@ gst_video_decoder_get_property (GObject * object, guint property_id, case PROP_AUTOMATIC_REQUEST_SYNC_POINT_FLAGS: g_value_set_flags (value, priv->automatic_request_sync_point_flags); break; + case PROP_DETECT_REORDERING: + g_value_set_boolean (value, priv->detect_reordering); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -1027,6 +1050,9 @@ gst_video_decoder_set_property (GObject * object, guint property_id, case PROP_AUTOMATIC_REQUEST_SYNC_POINT_FLAGS: priv->automatic_request_sync_point_flags = g_value_get_flags (value); break; + case PROP_DETECT_REORDERING: + priv->detect_reordering = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -1316,9 +1342,12 @@ gst_video_decoder_handle_missing_data_default (GstVideoDecoder * priv = decoder->priv; if (priv->automatic_request_sync_points) { - GstClockTime deadline = - gst_segment_to_running_time (&decoder->input_segment, GST_FORMAT_TIME, - timestamp); + GstClockTime deadline = timestamp; + + if (decoder->input_segment.format == GST_FORMAT_TIME) + deadline = + gst_segment_to_running_time (&decoder->input_segment, GST_FORMAT_TIME, + timestamp); GST_DEBUG_OBJECT (decoder, "Requesting sync point for missing data at running time %" @@ -3128,14 +3157,17 @@ gst_video_decoder_prepare_finish_frame (GstVideoDecoder * } } - if (GST_CLOCK_TIME_IS_VALID (priv->last_timestamp_out) && - frame->pts < priv->last_timestamp_out) { - GST_WARNING_OBJECT (decoder, - "decreasing timestamp (%" GST_TIME_FORMAT " < %" GST_TIME_FORMAT ")", - GST_TIME_ARGS (frame->pts), GST_TIME_ARGS (priv->last_timestamp_out)); - priv->reordered_output = TRUE; - /* make it a bit less weird downstream */ - frame->pts = priv->last_timestamp_out; + if (priv->detect_reordering + && GST_CLOCK_TIME_IS_VALID (priv->last_timestamp_out)) { + if (frame->pts < priv->last_timestamp_out) { + GST_WARNING_OBJECT (decoder, + "decreasing timestamp (%" GST_TIME_FORMAT " < %" + GST_TIME_FORMAT ")", + GST_TIME_ARGS (frame->pts), GST_TIME_ARGS (priv->last_timestamp_out)); + priv->reordered_output = TRUE; + /* make it a bit less weird downstream */ + frame->pts = priv->last_timestamp_out; + } } if (GST_CLOCK_TIME_IS_VALID (frame->pts)) @@ -5256,7 +5288,7 @@ gst_video_decoder_request_sync_point_internal (GstVideoDecoder * dec, "Requesting a new key-unit for frame with deadline %" GST_TIME_FORMAT, GST_TIME_ARGS (deadline)); fku = - gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, FALSE, + gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, TRUE, 0); priv->last_force_key_unit_time = deadline; } else { diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c index 2c269e1aa3b..78015735c74 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c @@ -458,6 +458,7 @@ gst_video_encoder_reset (GstVideoEncoder * encoder, gboolean hard) priv->presentation_frame_number = 0; priv->distance_from_sync = 0; + GST_OBJECT_LOCK (encoder); g_queue_clear_full (&priv->force_key_unit, (GDestroyNotify) forced_key_unit_event_free); priv->last_force_key_unit_request = GST_CLOCK_TIME_NONE; @@ -465,7 +466,6 @@ gst_video_encoder_reset (GstVideoEncoder * encoder, gboolean hard) priv->drained = TRUE; - GST_OBJECT_LOCK (encoder); priv->bytes = 0; priv->time = 0; GST_OBJECT_UNLOCK (encoder); @@ -2517,6 +2517,7 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder, gboolean send_headers = FALSE; gboolean key_unit = FALSE; gboolean discont = FALSE; + gboolean force_key_unit_present; GstBuffer *buffer; g_return_val_if_fail (frame, GST_FLOW_ERROR); @@ -2551,7 +2552,11 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder, priv->processed++; - if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame) && priv->force_key_unit.head) + GST_OBJECT_LOCK (encoder); + force_key_unit_present = priv->force_key_unit.head != NULL; + GST_OBJECT_UNLOCK (encoder); + + if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame) && force_key_unit_present) gst_video_encoder_send_key_unit_unlocked (encoder, frame, &send_headers); if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame) @@ -2579,12 +2584,6 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder, GST_BUFFER_DTS (frame->output_buffer) = frame->dts; GST_BUFFER_DURATION (frame->output_buffer) = frame->duration; - /* At this stage we have a full frame in subframe use case , - * let's mark it to enabled some latency optimization - * in some uses cases like RTP. */ - - GST_BUFFER_FLAG_SET (frame->output_buffer, GST_VIDEO_BUFFER_FLAG_MARKER); - GST_OBJECT_LOCK (encoder); /* update rate estimate */ priv->bytes += gst_buffer_get_size (frame->output_buffer); @@ -2675,6 +2674,7 @@ gst_video_encoder_finish_subframe (GstVideoEncoder * encoder, gboolean discont = FALSE; gboolean send_headers = FALSE; gboolean key_unit = FALSE; + gboolean force_key_unit_present; g_return_val_if_fail (frame, GST_FLOW_ERROR); g_return_val_if_fail (frame->output_buffer, GST_FLOW_ERROR); @@ -2697,7 +2697,11 @@ gst_video_encoder_finish_subframe (GstVideoEncoder * encoder, if (ret != GST_FLOW_OK) goto done; - if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame) && priv->force_key_unit.head) + GST_OBJECT_LOCK (encoder); + force_key_unit_present = priv->force_key_unit.head != NULL; + GST_OBJECT_UNLOCK (encoder); + + if (GST_VIDEO_CODEC_FRAME_IS_SYNC_POINT (frame) && force_key_unit_present) gst_video_encoder_send_key_unit_unlocked (encoder, frame, &send_headers); /* Push pending events only for the first subframe ie segment event. diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideopool.c b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideopool.c index fd4198d79f8..ccbbff42be7 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideopool.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideopool.c @@ -170,31 +170,14 @@ video_buffer_pool_set_config (GstBufferPool * pool, GstStructure * config) GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT); if (priv->need_alignment && priv->add_videometa) { - guint max_align, n; - + /* get and apply the alignment to the info */ gst_buffer_pool_config_get_video_alignment (config, &priv->video_align); - /* ensure GstAllocationParams alignment is compatible with video alignment */ - max_align = priv->params.align; - for (n = 0; n < GST_VIDEO_MAX_PLANES; ++n) - max_align |= priv->video_align.stride_align[n]; - - for (n = 0; n < GST_VIDEO_MAX_PLANES; ++n) - priv->video_align.stride_align[n] = max_align; - /* apply the alignment to the info */ if (!gst_video_info_align (&info, &priv->video_align)) goto failed_to_align; gst_buffer_pool_config_set_video_alignment (config, &priv->video_align); - - if (priv->params.align < max_align) { - GST_WARNING_OBJECT (pool, "allocation params alignment %u is smaller " - "than the max specified video stride alignment %u, fixing", - (guint) priv->params.align, max_align); - priv->params.align = max_align; - gst_buffer_pool_config_set_allocator (config, allocator, &priv->params); - } } info.size = MAX (size, info.size); priv->info = info; diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/video-info.c b/subprojects/gst-plugins-base/gst-libs/gst/video/video-info.c index d140dfabb06..948337eb3e9 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/video-info.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/video-info.c @@ -197,7 +197,7 @@ validate_colorimetry (GstVideoInfo * info) if (GST_VIDEO_FORMAT_INFO_IS_YUV (finfo) && info->colorimetry.matrix == GST_VIDEO_COLOR_MATRIX_UNKNOWN) { - GST_WARNING ("Need to specify a color matrix when using YUV format (%s)", + GST_INFO ("Need to specify a color matrix when using YUV format (%s)", finfo->name); return FALSE; } @@ -513,7 +513,7 @@ gst_video_info_from_caps (GstVideoInfo * info, const GstCaps * caps) GST_WARNING ("unparsable colorimetry, using default"); set_default_colorimetry (info); } else if (!validate_colorimetry (info)) { - GST_WARNING ("invalid colorimetry, using default"); + GST_INFO ("invalid colorimetry, using default"); set_default_colorimetry (info); } else { /* force RGB matrix for RGB formats */ diff --git a/subprojects/gst-plugins-base/gst/playback/gstdecodebin2.c b/subprojects/gst-plugins-base/gst/playback/gstdecodebin2.c index 2e8c943118a..4a8116481a8 100644 --- a/subprojects/gst-plugins-base/gst/playback/gstdecodebin2.c +++ b/subprojects/gst-plugins-base/gst/playback/gstdecodebin2.c @@ -3342,7 +3342,7 @@ gst_decode_chain_get_current_group (GstDecodeChain * chain) for (iter = chain->next_groups; iter; iter = g_list_next (iter)) { GstDecodeGroup *next_group = iter->data; - if (!next_group->overrun && !next_group->no_more_pads) { + if (next_group && !next_group->overrun && !next_group->no_more_pads) { group = next_group; break; } diff --git a/subprojects/gst-plugins-base/gst/playback/gstplaybin2.c b/subprojects/gst-plugins-base/gst/playback/gstplaybin2.c index 5a4ca4404c3..e1b8b9bf144 100644 --- a/subprojects/gst-plugins-base/gst/playback/gstplaybin2.c +++ b/subprojects/gst-plugins-base/gst/playback/gstplaybin2.c @@ -3024,11 +3024,11 @@ gst_play_bin_handle_message (GstBin * bin, GstMessage * msg) if (gst_uri_is_valid (location)) { uri = g_strdup (location); - } else { + } else if (group) { uri = gst_uri_join_strings (group->uri, location); } - if (g_strcmp0 (uri, group->uri)) { + if (uri && group && g_strcmp0 (uri, group->uri)) { GST_PLAY_BIN_LOCK (playbin); if (playbin->next_group && playbin->next_group->valid) { GST_DEBUG_OBJECT (playbin, diff --git a/subprojects/gst-plugins-base/gst/playback/gsturidecodebin.c b/subprojects/gst-plugins-base/gst/playback/gsturidecodebin.c index 27ec4f3c40e..67fa8a491e6 100644 --- a/subprojects/gst-plugins-base/gst/playback/gsturidecodebin.c +++ b/subprojects/gst-plugins-base/gst/playback/gsturidecodebin.c @@ -1305,11 +1305,11 @@ array_has_value (const gchar * values[], const gchar * value) } static gboolean -array_has_uri_value (const gchar * values[], const gchar * value) +array_has_uri_value (const gchar * values[], gint nelems, const gchar * value) { gint i; - for (i = 0; values[i]; i++) { + for (i = 0; i < nelems && values[i]; i++) { if (!g_ascii_strncasecmp (value, values[i], strlen (values[i]))) return TRUE; } @@ -1336,9 +1336,9 @@ static const gchar *adaptive_media[] = { "application/dash+xml", NULL }; -#define IS_STREAM_URI(uri) (array_has_uri_value (stream_uris, uri)) -#define IS_QUEUE_URI(uri) (array_has_uri_value (queue_uris, uri)) -#define IS_BLACKLISTED_URI(uri) (array_has_uri_value (blacklisted_uris, uri)) +#define IS_STREAM_URI(uri) (array_has_uri_value (stream_uris, G_N_ELEMENTS(stream_uris), uri)) +#define IS_QUEUE_URI(uri) (array_has_uri_value (queue_uris, G_N_ELEMENTS(queue_uris), uri)) +#define IS_BLACKLISTED_URI(uri) (array_has_uri_value (blacklisted_uris, G_N_ELEMENTS(blacklisted_uris), uri)) #define IS_ADAPTIVE_MEDIA(media) (array_has_value (adaptive_media, media)) /* diff --git a/subprojects/gst-plugins-base/gst/playback/gsturisourcebin.c b/subprojects/gst-plugins-base/gst/playback/gsturisourcebin.c index 8dcf371bdfd..ae5db1d8100 100644 --- a/subprojects/gst-plugins-base/gst/playback/gsturisourcebin.c +++ b/subprojects/gst-plugins-base/gst/playback/gsturisourcebin.c @@ -1443,11 +1443,11 @@ demuxer_pad_removed_cb (GstElement * element, GstPad * pad, /* helper function to lookup stuff in lists */ static gboolean -array_has_value (const gchar * values[], const gchar * value) +array_has_value (const gchar * values[], gint nelems, const gchar * value) { gint i; - for (i = 0; values[i]; i++) { + for (i = 0; i < nelems && values[i]; i++) { if (g_str_has_prefix (value, values[i])) return TRUE; } @@ -1455,11 +1455,11 @@ array_has_value (const gchar * values[], const gchar * value) } static gboolean -array_has_uri_value (const gchar * values[], const gchar * value) +array_has_uri_value (const gchar * values[], gint nelems, const gchar * value) { gint i; - for (i = 0; values[i]; i++) { + for (i = 0; i < nelems && values[i]; i++) { if (!g_ascii_strncasecmp (value, values[i], strlen (values[i]))) return TRUE; } @@ -1486,10 +1486,10 @@ static const gchar *adaptive_media[] = { "application/dash+xml", NULL }; -#define IS_STREAM_URI(uri) (array_has_uri_value (stream_uris, uri)) -#define IS_QUEUE_URI(uri) (array_has_uri_value (queue_uris, uri)) -#define IS_BLACKLISTED_URI(uri) (array_has_uri_value (blacklisted_uris, uri)) -#define IS_ADAPTIVE_MEDIA(media) (array_has_value (adaptive_media, media)) +#define IS_STREAM_URI(uri) (array_has_uri_value (stream_uris, G_N_ELEMENTS(stream_uris), uri)) +#define IS_QUEUE_URI(uri) (array_has_uri_value (queue_uris, G_N_ELEMENTS(queue_uris), uri)) +#define IS_BLACKLISTED_URI(uri) (array_has_uri_value (blacklisted_uris, G_N_ELEMENTS(blacklisted_uris), uri)) +#define IS_ADAPTIVE_MEDIA(media) (array_has_value (adaptive_media, G_N_ELEMENTS(adaptive_media), media)) /* * Generate and configure a source element. diff --git a/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c b/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c index f16b4753838..4cd8dcc5bf2 100644 --- a/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c +++ b/subprojects/gst-plugins-base/gst/videorate/gstvideorate.c @@ -100,7 +100,6 @@ enum #define DEFAULT_MAX_RATE G_MAXINT #define DEFAULT_RATE 1.0 #define DEFAULT_MAX_DUPLICATION_TIME 0 -#define DEFAULT_MAX_CLOSING_SEGMENT_DUPLICATION_DURATION GST_SECOND enum { @@ -116,8 +115,7 @@ enum PROP_AVERAGE_PERIOD, PROP_MAX_RATE, PROP_RATE, - PROP_MAX_DUPLICATION_TIME, - PROP_MAX_CLOSING_SEGMENT_DUPLICATION_DURATION + PROP_MAX_DUPLICATION_TIME }; static GstStaticPadTemplate gst_video_rate_src_template = @@ -297,27 +295,6 @@ gst_video_rate_class_init (GstVideoRateClass * klass) 0, G_MAXUINT64, DEFAULT_MAX_DUPLICATION_TIME, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - /** - * GstVideoRate:max-closing-segment-duplication-duration: - * - * Limits the maximum duration for which the last buffer is duplicated when - * finalizing a segment or on EOS. When receiving an EOS event or a new - * segment, videorate duplicates the last frame to close the configured - * segment (copying the last buffer until its #GstSegment.stop time (or - * #GstSegment.start time for reverse playback) is reached), this property - * ensures that it won't push buffers covering a duration longer than - * specified. - * - * Since: 1.22 - */ - g_object_class_install_property (object_class, - PROP_MAX_CLOSING_SEGMENT_DUPLICATION_DURATION, - g_param_spec_uint64 ("max-closing-segment-duplication-duration", - "Maximum closing segment duplication duration", - "Maximum duration of duplicated buffers to close current segment", 0, - G_MAXUINT64, DEFAULT_MAX_CLOSING_SEGMENT_DUPLICATION_DURATION, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - gst_element_class_set_static_metadata (element_class, "Video rate adjuster", "Filter/Effect/Video", "Drops/duplicates/adjusts timestamps on video frames to make a perfect stream", @@ -613,10 +590,26 @@ gst_video_rate_setcaps (GstBaseTransform * trans, GstCaps * in_caps, else videorate->wanted_diff = 0; -done: - if (ret) { - gst_caps_replace (&videorate->in_caps, in_caps); + /* PEXHACK + * drop-only needed due to: + * https://github.com/pexip/mcu/commit/f819d921773ab222ee0fb950e9808c48b3794156 + */ + { + gint in_fps = + (videorate->from_rate_numerator * 1000) / + videorate->from_rate_denominator; + gint out_fps = + (videorate->to_rate_numerator * 1000) / videorate->to_rate_denominator; + if (in_fps >= out_fps) + videorate->drop_only = TRUE; } +done: + /* After a setcaps, our caps may have changed. In that case, we can't use + * the old buffer, if there was one (it might have different dimensions) */ + GST_DEBUG_OBJECT (videorate, "swapping old buffers"); + gst_video_rate_swap_prev (videorate, NULL, GST_CLOCK_TIME_NONE); + videorate->last_ts = GST_CLOCK_TIME_NONE; + videorate->average = 0; return ret; @@ -629,7 +622,7 @@ gst_video_rate_setcaps (GstBaseTransform * trans, GstCaps * in_caps, } static void -gst_video_rate_reset (GstVideoRate * videorate, gboolean on_flush) +gst_video_rate_reset (GstVideoRate * videorate) { GST_DEBUG_OBJECT (videorate, "resetting internal variables"); @@ -644,10 +637,6 @@ gst_video_rate_reset (GstVideoRate * videorate, gboolean on_flush) videorate->discont = TRUE; videorate->average = 0; videorate->force_variable_rate = FALSE; - if (!on_flush) { - /* Do not clear caps on flush events as those are still valid */ - gst_clear_caps (&videorate->in_caps); - } gst_video_rate_swap_prev (videorate, NULL, 0); gst_segment_init (&videorate->segment, GST_FORMAT_TIME); @@ -656,7 +645,7 @@ gst_video_rate_reset (GstVideoRate * videorate, gboolean on_flush) static void gst_video_rate_init (GstVideoRate * videorate) { - gst_video_rate_reset (videorate, FALSE); + gst_video_rate_reset (videorate); videorate->silent = DEFAULT_SILENT; videorate->new_pref = DEFAULT_NEW_PREF; videorate->drop_only = DEFAULT_DROP_ONLY; @@ -666,8 +655,6 @@ gst_video_rate_init (GstVideoRate * videorate) videorate->rate = DEFAULT_RATE; videorate->pending_rate = DEFAULT_RATE; videorate->max_duplication_time = DEFAULT_MAX_DUPLICATION_TIME; - videorate->max_closing_segment_duplication_duration = - DEFAULT_MAX_CLOSING_SEGMENT_DUPLICATION_DURATION; videorate->from_rate_numerator = 0; videorate->from_rate_denominator = 0; @@ -680,7 +667,7 @@ gst_video_rate_init (GstVideoRate * videorate) /* @outbuf: (transfer full) needs to be writable */ static GstFlowReturn gst_video_rate_push_buffer (GstVideoRate * videorate, GstBuffer * outbuf, - gboolean duplicate, GstClockTime next_intime, gboolean invalid_duration) + gboolean duplicate, GstClockTime next_intime) { GstFlowReturn res; GstClockTime push_ts; @@ -738,7 +725,7 @@ gst_video_rate_push_buffer (GstVideoRate * videorate, GstBuffer * outbuf, videorate->to_rate_denominator * GST_SECOND, videorate->to_rate_numerator); GST_BUFFER_DURATION (outbuf) = videorate->next_ts - push_ts; - } else if (!invalid_duration) { + } else { /* There must always be a valid duration on prevbuf if rate > 0, * it is ensured in the transform_ip function */ g_assert (GST_BUFFER_PTS_IS_VALID (outbuf)); @@ -768,7 +755,7 @@ gst_video_rate_push_buffer (GstVideoRate * videorate, GstBuffer * outbuf, /* flush the oldest buffer */ static GstFlowReturn gst_video_rate_flush_prev (GstVideoRate * videorate, gboolean duplicate, - GstClockTime next_intime, gboolean invalid_duration) + GstClockTime next_intime) { GstBuffer *outbuf; @@ -779,8 +766,7 @@ gst_video_rate_flush_prev (GstVideoRate * videorate, gboolean duplicate, /* make sure we can write to the metadata */ outbuf = gst_buffer_make_writable (outbuf); - return gst_video_rate_push_buffer (videorate, outbuf, duplicate, next_intime, - invalid_duration); + return gst_video_rate_push_buffer (videorate, outbuf, duplicate, next_intime); /* WARNINGS */ eos_before_buffers: @@ -795,14 +781,9 @@ gst_video_rate_swap_prev (GstVideoRate * videorate, GstBuffer * buffer, gint64 time) { GST_LOG_OBJECT (videorate, "swap_prev: storing buffer %p in prev", buffer); - - gst_buffer_replace (&videorate->prevbuf, buffer); - /* Ensure that ->prev_caps always match ->prevbuf */ - if (!buffer) - gst_caps_replace (&videorate->prev_caps, NULL); - else if (videorate->prev_caps != videorate->in_caps) - gst_caps_replace (&videorate->prev_caps, videorate->in_caps); - + if (videorate->prevbuf) + gst_buffer_unref (videorate->prevbuf); + videorate->prevbuf = buffer != NULL ? gst_buffer_ref (buffer) : NULL; videorate->prev_ts = time; } @@ -818,136 +799,7 @@ gst_video_rate_notify_duplicate (GstVideoRate * videorate) g_object_notify_by_pspec ((GObject *) videorate, pspec_duplicate); } -static gboolean -gst_video_rate_check_duplicate_to_close_segment (GstVideoRate * videorate, - GstClockTime last_input_ts, gboolean is_first) -{ - GstClockTime next_stream_time = videorate->next_ts - videorate->segment.base; - GstClockTime max_closing_segment_duplication_duration = - videorate->max_closing_segment_duplication_duration; - - if (!GST_CLOCK_TIME_IS_VALID (videorate->next_ts)) - return FALSE; - - if (videorate->segment.rate > 0.0) { - - if (!GST_CLOCK_TIME_IS_VALID (videorate->segment.stop)) { - /* Ensure that if no 'stop' is set, we push the last frame anyway */ - return is_first; - } - - if (next_stream_time >= videorate->segment.stop) - return FALSE; - - if (GST_CLOCK_TIME_IS_VALID (max_closing_segment_duplication_duration)) { - if (last_input_ts > videorate->next_ts) - return TRUE; - - return (videorate->next_ts - last_input_ts < - max_closing_segment_duplication_duration); - } - - return TRUE; - } - - /* Reverse playback */ - - if (!GST_CLOCK_TIME_IS_VALID (videorate->segment.start)) { - /* Ensure that if no 'start' is set, we push the last frame anyway */ - return is_first; - } - - if (next_stream_time < videorate->segment.start) - return FALSE; - - if (GST_CLOCK_TIME_IS_VALID (max_closing_segment_duplication_duration)) { - if (last_input_ts < videorate->next_ts) - return TRUE; - - return (last_input_ts - videorate->next_ts < - max_closing_segment_duplication_duration); - } - - return TRUE; -} - -static gint -gst_video_rate_duplicate_to_close_segment (GstVideoRate * videorate) -{ - gint count = 0; - GstFlowReturn res; - GstClockTime last_input_ts = videorate->prev_ts; - - if (videorate->drop_only) - return count; - - if (!videorate->prevbuf) { - GST_INFO_OBJECT (videorate, "got EOS before any buffer was received"); - - return count; - } - - GST_DEBUG_OBJECT (videorate, "Pushing buffers to close segment"); - - res = GST_FLOW_OK; - /* fill up to the end of current segment */ - while (res == GST_FLOW_OK - && gst_video_rate_check_duplicate_to_close_segment (videorate, - last_input_ts, count < 1)) { - res = - gst_video_rate_flush_prev (videorate, count > 0, GST_CLOCK_TIME_NONE, - FALSE); - - count++; - } - GST_DEBUG_OBJECT (videorate, "----> Pushed %d buffers to close segment", - count); - - return count; -} - -/* WORKAROUND: This works around BaseTransform limitation as instead of rolling - * back caps, we should be able to push caps only when we are sure we are ready - * to do so. Right now, BaseTransform doesn't let us do anything like that - * so we rollback to previous caps when strictly required (though we now it - * might not be so safe). - * - * To be used only when wanting to 'close' a segment, this function will reset - * caps to previous caps, which will match the content of `prevbuf` in that case - * - * Returns: The previous GstCaps if we rolled back to previous buffers, NULL - * otherwise. - * - * NOTE: When some caps are returned, we should reset them back after - * closing the segment is done. - */ -static GstCaps * -gst_video_rate_rollback_to_prev_caps_if_needed (GstVideoRate * videorate) -{ - GstCaps *prev_caps = NULL; - - if (videorate->prev_caps && videorate->prev_caps != videorate->in_caps) { - if (videorate->in_caps) - prev_caps = gst_caps_ref (videorate->in_caps); - - if (!gst_pad_send_event (GST_BASE_TRANSFORM_SINK_PAD (videorate), - gst_event_new_caps (videorate->prev_caps) - )) { - - GST_WARNING_OBJECT (videorate, "Could not send previous caps to close " - " segment, not closing it"); - - gst_video_rate_swap_prev (videorate, NULL, GST_CLOCK_TIME_NONE); - videorate->last_ts = GST_CLOCK_TIME_NONE; - videorate->average = 0; - } - - gst_clear_caps (&videorate->prev_caps); - } - - return prev_caps; -} - +#define MAGIC_LIMIT 25 static gboolean gst_video_rate_sink_event (GstBaseTransform * trans, GstEvent * event) { @@ -960,27 +812,62 @@ gst_video_rate_sink_event (GstBaseTransform * trans, GstEvent * event) { GstSegment segment; gint seqnum; - GstCaps *rolled_back_caps; + GstClockTime base_ts, next_ts; + gboolean reset_base_ts = FALSE; gst_event_copy_segment (event, &segment); if (segment.format != GST_FORMAT_TIME) goto format_error; + GST_DEBUG_OBJECT (videorate, "handle SEGMENT event"); + + /* We just want to update the accumulated stream_time */ + segment.start = (gint64) (segment.start / videorate->rate); segment.position = (gint64) (segment.position / videorate->rate); if (GST_CLOCK_TIME_IS_VALID (segment.stop)) segment.stop = (gint64) (segment.stop / videorate->rate); segment.time = (gint64) (segment.time / videorate->rate); + base_ts = gst_segment_position_from_running_time (&segment, + GST_FORMAT_TIME, + gst_segment_to_running_time (&videorate->segment, GST_FORMAT_TIME, + videorate->base_ts)); + next_ts = gst_segment_position_from_running_time (&segment, + GST_FORMAT_TIME, + gst_segment_to_running_time (&videorate->segment, GST_FORMAT_TIME, + videorate->next_ts)); - if (!gst_segment_is_equal (&segment, &videorate->segment)) { - rolled_back_caps = - gst_video_rate_rollback_to_prev_caps_if_needed (videorate); + /* Reset if the segment is discontinuous */ + if (next_ts == GST_CLOCK_TIME_NONE) { + reset_base_ts = TRUE; /* close up the previous segment, if appropriate */ if (videorate->prevbuf) { - /* fill up to the end of current segment */ - gint count = gst_video_rate_duplicate_to_close_segment (videorate); + gint count = 0; + GstFlowReturn res; + + res = GST_FLOW_OK; + /* fill up to the end of current segment, + * or only send out the stored buffer if there is no specific stop. + * regardless, prevent going loopy in strange cases */ + while (res == GST_FLOW_OK && count <= MAGIC_LIMIT + && !videorate->drop_only + && ((videorate->segment.rate > 0.0 + && GST_CLOCK_TIME_IS_VALID (videorate->segment.stop) + && GST_CLOCK_TIME_IS_VALID (videorate->next_ts) + && videorate->next_ts - videorate->segment.base < + videorate->segment.stop) || (videorate->segment.rate < 0.0 + && GST_CLOCK_TIME_IS_VALID (videorate->segment.start) + && GST_CLOCK_TIME_IS_VALID (videorate->next_ts) + && videorate->next_ts - videorate->segment.base >= + videorate->segment.start) + || count < 1)) { + res = + gst_video_rate_flush_prev (videorate, count > 0, + GST_CLOCK_TIME_NONE); + count++; + } if (count > 1) { videorate->dup += count - 1; if (!videorate->silent) @@ -989,39 +876,21 @@ gst_video_rate_sink_event (GstBaseTransform * trans, GstEvent * event) /* clean up for the new one; _chain will resume from the new start */ gst_video_rate_swap_prev (videorate, NULL, 0); } - - if (rolled_back_caps) { - GST_DEBUG_OBJECT (videorate, - "Resetting rolled back caps %" GST_PTR_FORMAT, rolled_back_caps); - if (!gst_pad_send_event (GST_BASE_TRANSFORM_SINK_PAD (videorate), - gst_event_new_caps (rolled_back_caps) - )) { - - GST_WARNING_OBJECT (videorate, - "Could not resend caps after closing " " segment"); - - GST_ELEMENT_ERROR (videorate, CORE, NEGOTIATION, - ("Could not resend caps after closing segment"), (NULL)); - gst_caps_unref (rolled_back_caps); - - return FALSE; - } - - gst_caps_unref (rolled_back_caps); - } + } else if (base_ts == GST_CLOCK_TIME_NONE) { + reset_base_ts = TRUE; } - videorate->base_ts = 0; - videorate->out_frame_count = 0; - videorate->next_ts = GST_CLOCK_TIME_NONE; - - /* We just want to update the accumulated stream_time */ + if (reset_base_ts) { + base_ts = 0; + videorate->out_frame_count = 0; + } + videorate->next_ts = next_ts; + videorate->base_ts = base_ts; gst_segment_copy_into (&segment, &videorate->segment); GST_DEBUG_OBJECT (videorate, "updated segment: %" GST_SEGMENT_FORMAT, &videorate->segment); - seqnum = gst_event_get_seqnum (event); gst_event_unref (event); event = gst_event_new_segment (&segment); @@ -1033,68 +902,56 @@ gst_video_rate_sink_event (GstBaseTransform * trans, GstEvent * event) case GST_EVENT_EOS:{ gint count = 0; GstFlowReturn res = GST_FLOW_OK; - GstCaps *rolled_back_caps; GST_DEBUG_OBJECT (videorate, "Got %s", gst_event_type_get_name (GST_EVENT_TYPE (event))); - rolled_back_caps = - gst_video_rate_rollback_to_prev_caps_if_needed (videorate); - /* If the segment has a stop position, fill the segment */ if (GST_CLOCK_TIME_IS_VALID (videorate->segment.stop)) { - /* fill up to the end of current segment */ - count = gst_video_rate_duplicate_to_close_segment (videorate); + /* fill up to the end of current segment, + * or only send out the stored buffer if there is no specific stop. + * regardless, prevent going loopy in strange cases */ + while (res == GST_FLOW_OK && count <= MAGIC_LIMIT + && !videorate->drop_only + && ((videorate->segment.rate > 0.0 + && GST_CLOCK_TIME_IS_VALID (videorate->segment.stop) + && GST_CLOCK_TIME_IS_VALID (videorate->next_ts) + && videorate->next_ts - videorate->segment.base < + videorate->segment.stop) || (videorate->segment.rate < 0.0 + && GST_CLOCK_TIME_IS_VALID (videorate->segment.start) + && GST_CLOCK_TIME_IS_VALID (videorate->next_ts) + && videorate->next_ts - videorate->segment.base >= + videorate->segment.start) + )) { + res = gst_video_rate_flush_prev (videorate, count > 0, + GST_CLOCK_TIME_NONE); + count++; + } } else if (!videorate->drop_only && videorate->prevbuf) { /* Output at least one frame but if the buffer duration is valid, output * enough frames to use the complete buffer duration */ if (GST_BUFFER_DURATION_IS_VALID (videorate->prevbuf)) { - GstClockTime end_ts, duration = - GST_BUFFER_DURATION (videorate->prevbuf); + GstClockTime end_ts = + videorate->next_ts + GST_BUFFER_DURATION (videorate->prevbuf); - if (GST_CLOCK_TIME_IS_VALID - (videorate->max_closing_segment_duplication_duration)) - duration = - MIN (videorate->max_closing_segment_duplication_duration, - duration); - - end_ts = videorate->next_ts + duration; - while (res == GST_FLOW_OK && ((videorate->segment.rate > 0.0 + while (res == GST_FLOW_OK && count <= MAGIC_LIMIT && + ((videorate->segment.rate > 0.0 && GST_CLOCK_TIME_IS_VALID (videorate->segment.stop) && GST_CLOCK_TIME_IS_VALID (videorate->next_ts) && videorate->next_ts - videorate->segment.base < end_ts) || count < 1)) { res = gst_video_rate_flush_prev (videorate, count > 0, - GST_CLOCK_TIME_NONE, FALSE); + GST_CLOCK_TIME_NONE); count++; } } else { - /* allow the duration to be invalid as there is no way to infer it if we - * received a single buffer and not output framerate was set. */ res = - gst_video_rate_flush_prev (videorate, FALSE, GST_CLOCK_TIME_NONE, - TRUE); + gst_video_rate_flush_prev (videorate, FALSE, GST_CLOCK_TIME_NONE); count = 1; } } - if (rolled_back_caps) { - GST_DEBUG_OBJECT (videorate, - "Resetting rolled back caps %" GST_PTR_FORMAT, rolled_back_caps); - - if (!gst_pad_send_event (GST_BASE_TRANSFORM_SINK_PAD (videorate), - gst_event_new_caps (rolled_back_caps) - )) { - - /* Not erroring out on EOS as it won't be too bad in any case */ - GST_WARNING_OBJECT (videorate, "Could not resend caps after closing " - " segment on EOS (ignoring the error)"); - } - - gst_caps_unref (rolled_back_caps); - } - if (count > 1) { videorate->dup += count - 1; if (!videorate->silent) @@ -1111,7 +968,7 @@ gst_video_rate_sink_event (GstBaseTransform * trans, GstEvent * event) case GST_EVENT_FLUSH_STOP: /* also resets the segment */ GST_DEBUG_OBJECT (videorate, "Got FLUSH_STOP"); - gst_video_rate_reset (videorate, TRUE); + gst_video_rate_reset (videorate); break; case GST_EVENT_GAP: /* no gaps after videorate, ignore the event */ @@ -1578,14 +1435,12 @@ gst_video_rate_do_max_duplicate (GstVideoRate * videorate, GstBuffer * buffer, * previous buffer */ if (videorate->segment.rate < 0.0) { while (videorate->next_ts > prevtime) { - gst_video_rate_flush_prev (videorate, *count > 0, GST_CLOCK_TIME_NONE, - FALSE); + gst_video_rate_flush_prev (videorate, *count > 0, GST_CLOCK_TIME_NONE); *count += 1; } } else { while (videorate->next_ts <= prevtime) { - gst_video_rate_flush_prev (videorate, *count > 0, GST_CLOCK_TIME_NONE, - FALSE); + gst_video_rate_flush_prev (videorate, *count > 0, GST_CLOCK_TIME_NONE); *count += 1; } } @@ -1647,18 +1502,6 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer) videorate = GST_VIDEO_RATE (trans); - if (videorate->prev_caps != videorate->in_caps) { - /* After caps where set we didn't reset the state so we could close - * the segment from previous caps if necessary, we got a buffer after the - * new caps so we can reset now */ - GST_DEBUG_OBJECT (videorate, "Clearing old buffers now that we had a buffer" - " after receiving caps"); - gst_video_rate_swap_prev (videorate, NULL, GST_CLOCK_TIME_NONE); - gst_clear_caps (&videorate->prev_caps); - videorate->last_ts = GST_CLOCK_TIME_NONE; - videorate->average = 0; - } - /* make sure the denominators are not 0 */ if (videorate->from_rate_denominator == 0 || videorate->to_rate_denominator == 0) @@ -1696,6 +1539,10 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer) goto invalid_buffer; } + if (!gst_segment_clip (&videorate->segment, GST_FORMAT_TIME, in_ts, + GST_CLOCK_TIME_NONE, NULL, NULL)) + goto outside_segment; + /* get the time of the next expected buffer timestamp, we use this when the * next buffer has -1 as a timestamp */ last_ts = videorate->last_ts; @@ -1712,15 +1559,23 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer) /* we need to have two buffers to compare */ if (videorate->prevbuf == NULL || videorate->drop_only) { - /* We can calculate the duration of the buffer here if not given for - * reverse playback. We need this later */ - if (videorate->segment.rate < 0.0 && !GST_BUFFER_DURATION_IS_VALID (buffer)) { - /* As we require valid timestamps all the time for reverse playback, we either - * have a valid last_ts or we're at the very first buffer. */ - if (!GST_CLOCK_TIME_IS_VALID (last_ts)) - GST_BUFFER_DURATION (buffer) = videorate->segment.stop - in_ts; - else - GST_BUFFER_DURATION (buffer) = last_ts - in_ts; + /* We can calculate the duration of the buffer here if not given */ + if (!GST_BUFFER_DURATION_IS_VALID (buffer)) { + /* For reverse playback. We need this later */ + if (videorate->segment.rate < 0.0) { + /* As we require valid timestamps all the time for reverse playback, we either + * have a valid last_ts or we're at the very first buffer. */ + if (!GST_CLOCK_TIME_IS_VALID (last_ts)) + GST_BUFFER_DURATION (buffer) = videorate->segment.stop - in_ts; + else + GST_BUFFER_DURATION (buffer) = last_ts - in_ts; + } else if (videorate->to_rate_numerator) { + GST_BUFFER_DURATION (buffer) = gst_util_uint64_scale (1, + videorate->to_rate_denominator * GST_SECOND, + videorate->to_rate_numerator); + } else { + goto invalid_buffer_duration; + } } gst_video_rate_swap_prev (videorate, buffer, intime); @@ -1777,7 +1632,7 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer) * GstBaseTransform can get its reference back. */ if ((r = gst_video_rate_push_buffer (videorate, gst_buffer_ref (buffer), FALSE, - GST_CLOCK_TIME_NONE, FALSE)) != GST_FLOW_OK) { + GST_CLOCK_TIME_NONE)) != GST_FLOW_OK) { res = r; goto done; } @@ -1907,7 +1762,7 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer) /* on error the _flush function posted a warning already */ if ((r = gst_video_rate_flush_prev (videorate, - count > 1, intime, FALSE)) != GST_FLOW_OK) { + count > 1, intime)) != GST_FLOW_OK) { res = r; goto done; } @@ -1964,19 +1819,34 @@ gst_video_rate_transform_ip (GstBaseTransform * trans, GstBuffer * buffer) res = GST_BASE_TRANSFORM_FLOW_DROPPED; goto done; } + +invalid_buffer_duration: + { + GST_WARNING_OBJECT (videorate, + "Got first buffer with GST_CLOCK_TIME_NONE duration, discarding it"); + res = GST_BASE_TRANSFORM_FLOW_DROPPED; + goto done; + } + +outside_segment: + { + GST_WARNING_OBJECT (videorate, "Got buffer outide segment, discarding it"); + res = GST_BASE_TRANSFORM_FLOW_DROPPED; + goto done; + } } static gboolean gst_video_rate_start (GstBaseTransform * trans) { - gst_video_rate_reset (GST_VIDEO_RATE (trans), FALSE); + gst_video_rate_reset (GST_VIDEO_RATE (trans)); return TRUE; } static gboolean gst_video_rate_stop (GstBaseTransform * trans) { - gst_video_rate_reset (GST_VIDEO_RATE (trans), FALSE); + gst_video_rate_reset (GST_VIDEO_RATE (trans)); return TRUE; } @@ -2030,10 +1900,6 @@ gst_video_rate_set_property (GObject * object, case PROP_MAX_DUPLICATION_TIME: videorate->max_duplication_time = g_value_get_uint64 (value); break; - case PROP_MAX_CLOSING_SEGMENT_DUPLICATION_DURATION: - videorate->max_closing_segment_duplication_duration = - g_value_get_uint64 (value); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -2096,10 +1962,6 @@ gst_video_rate_get_property (GObject * object, case PROP_MAX_DUPLICATION_TIME: g_value_set_uint64 (value, videorate->max_duplication_time); break; - case PROP_MAX_CLOSING_SEGMENT_DUPLICATION_DURATION: - g_value_set_uint64 (value, - videorate->max_closing_segment_duplication_duration); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; diff --git a/subprojects/gst-plugins-base/meson.build b/subprojects/gst-plugins-base/meson.build index 20325085fa8..28e38262d29 100644 --- a/subprojects/gst-plugins-base/meson.build +++ b/subprojects/gst-plugins-base/meson.build @@ -79,9 +79,16 @@ if cc.get_id() == 'msvc' '/we4053', # one void operand for '?:' '/we4062', # enumerator 'identifier' in switch of enum 'enumeration' is not handled '/we4098', # 'function' : void function returning a value + ]) + + # add these warnings only if checks are enabled, + # if the checks are not built, it will leave behind some "unused" variables + if get_option('glib-checks').enabled() + msvc_args += cc.get_supported_arguments([ '/we4101', # 'identifier' : unreferenced local variable '/we4189', # 'identifier' : local variable is initialized but not referenced - ]) + ]) + endif endif add_project_arguments(msvc_args, language: ['c', 'cpp']) # Disable SAFESEH with MSVC for plugins and libs that use external deps that diff --git a/subprojects/gst-plugins-base/tests/check/libs/audiodecoder.c b/subprojects/gst-plugins-base/tests/check/libs/audiodecoder.c index 8f69f53208d..96059c2732e 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/audiodecoder.c +++ b/subprojects/gst-plugins-base/tests/check/libs/audiodecoder.c @@ -86,7 +86,7 @@ G_DEFINE_TYPE (GstAudioDecoderTester, gst_audio_decoder_tester, GST_TYPE_AUDIO_DECODER); static gboolean -gst_audio_decoder_tester_start (GstAudioDecoder * dec) +gst_audio_decoder_tester_start (G_GNUC_UNUSED GstAudioDecoder * dec) { return TRUE; } @@ -103,7 +103,8 @@ gst_audio_decoder_tester_stop (GstAudioDecoder * dec) } static void -gst_audio_decoder_tester_flush (GstAudioDecoder * dec, gboolean hard) +gst_audio_decoder_tester_flush (G_GNUC_UNUSED GstAudioDecoder * dec, + G_GNUC_UNUSED gboolean hard) { } @@ -137,11 +138,16 @@ gst_audio_decoder_tester_handle_frame (GstAudioDecoder * dec, GstFlowReturn ret = GST_FLOW_OK; gboolean do_plc = gst_audio_decoder_get_plc (dec) && gst_audio_decoder_get_plc_aware (dec); + gboolean is_dtx_frame; if (buffer == NULL || (!do_plc && gst_buffer_get_size (buffer) == 0)) return GST_FLOW_OK; gst_buffer_ref (buffer); + + /* for testing purposes, 1byte size buffers signals the start of DTX */ + is_dtx_frame = (gst_buffer_get_size (buffer) == 1); + if (tester->setoutputformat_on_decoding) { GstCaps *caps; GstAudioInfo info; @@ -157,7 +163,7 @@ gst_audio_decoder_tester_handle_frame (GstAudioDecoder * dec, if ((tester->delay_decoding && tester->prev_buf != NULL) || !tester->delay_decoding) { gsize buf_num = tester->delay_decoding ? 2 : 1; - gint i; + gsize i; for (i = 0; i != buf_num; ++i) { GstBuffer *cur_buf = buf_num == 1 || i != 0 ? buffer : tester->prev_buf; @@ -168,7 +174,7 @@ gst_audio_decoder_tester_handle_frame (GstAudioDecoder * dec, g_assert (size == sizeof (guint64)); data = g_malloc0 (size); - if (map.size) { + if (map.size && !is_dtx_frame) { g_assert_cmpint (map.size, >=, sizeof (guint64)); memcpy (data, map.data, sizeof (guint64)); } @@ -188,6 +194,9 @@ gst_audio_decoder_tester_handle_frame (GstAudioDecoder * dec, tester->delay_decoding = FALSE; } + if (is_dtx_frame) + gst_audio_decoder_start_dtx (dec); + if (tester->prev_buf) gst_buffer_unref (tester->prev_buf); tester->prev_buf = NULL; @@ -226,7 +235,7 @@ gst_audio_decoder_tester_class_init (GstAudioDecoderTesterClass * klass) } static void -gst_audio_decoder_tester_init (GstAudioDecoderTester * tester) +gst_audio_decoder_tester_init (G_GNUC_UNUSED GstAudioDecoderTester * tester) { } @@ -271,6 +280,52 @@ create_test_buffer (guint64 num) return buffer; } +static float +get_avg_buf_power (float *data, gint samples) +{ + float sum_power = 0.0f; + for (int s = 0; s < samples; s++) + sum_power += data[s] * data[s]; + + return sum_power / (float) samples; +} + +static void +fail_unless_silent_buffer (GstBuffer * buf) +{ + GstMapInfo map; + fail_unless (gst_buffer_map (buf, &map, GST_MAP_READ)); + float power = get_avg_buf_power ((float *) map.data, + (guint) map.size / sizeof (float)); + fail_unless_equals_float (0.0f, power); + gst_buffer_unmap (buf, &map); + gst_buffer_unref (buf); +} + +static void +fail_unless_silent_buffers (GstHarness * h, gint count, GstClockTime pts) +{ + for (gint i = 0; i < count; i++) { + GstBuffer *buf; + + gst_harness_crank_single_clock_wait (h); + buf = gst_harness_pull (h); + pts += GST_BUFFER_DURATION (buf); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); + fail_unless_silent_buffer (buf); + } +} + +static GstBuffer * +create_dtx_buffer (guint64 num) +{ + /* 1byte buffer signals DTX */ + GstBuffer *buf = create_test_buffer (num); + gst_buffer_set_size (buf, 1); + gst_buffer_memset (buf, 0, 0, 1); + return buf; +} + #define NUM_BUFFERS 10 GST_START_TEST (audiodecoder_playback) @@ -804,7 +859,8 @@ GST_END_TEST; #define GETCAPS_CAPS_STR "audio/x-test-custom, somefield=(string)getcaps" static GstCaps * -_custom_audio_decoder_getcaps (GstAudioDecoder * dec, GstCaps * filter) +_custom_audio_decoder_getcaps (G_GNUC_UNUSED GstAudioDecoder * dec, + G_GNUC_UNUSED GstCaps * filter) { return gst_caps_from_string (GETCAPS_CAPS_STR); } @@ -1063,16 +1119,16 @@ GST_START_TEST (audiodecoder_plc_on_gap_event) pts = gst_util_uint64_scale_round (0, GST_SECOND, TEST_MSECS_PER_SAMPLE); gst_harness_push (h, create_test_buffer (0)); buf = gst_harness_pull (h); - fail_unless_equals_int (pts, GST_BUFFER_PTS (buf)); - fail_unless_equals_int (dur, GST_BUFFER_DURATION (buf)); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); fail_unless (GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DISCONT)); gst_buffer_unref (buf); pts = gst_util_uint64_scale_round (1, GST_SECOND, TEST_MSECS_PER_SAMPLE); gst_harness_push_event (h, gst_event_new_gap (pts, dur)); buf = gst_harness_pull (h); - fail_unless_equals_int (pts, GST_BUFFER_PTS (buf)); - fail_unless_equals_int (dur, GST_BUFFER_DURATION (buf)); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); fail_unless (!GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DISCONT)); gst_buffer_unref (buf); @@ -1081,8 +1137,8 @@ GST_START_TEST (audiodecoder_plc_on_gap_event) GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DISCONT); gst_harness_push (h, buf); buf = gst_harness_pull (h); - fail_unless_equals_int (pts, GST_BUFFER_PTS (buf)); - fail_unless_equals_int (dur, GST_BUFFER_DURATION (buf)); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); fail_unless (!GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DISCONT)); gst_buffer_unref (buf); gst_harness_teardown (h); @@ -1106,8 +1162,8 @@ GST_START_TEST (audiodecoder_plc_on_gap_event_with_delay) pts0 = gst_util_uint64_scale_round (0, GST_SECOND, TEST_MSECS_PER_SAMPLE);; gst_harness_push (h, create_test_buffer (0)); buf = gst_harness_pull (h); - fail_unless_equals_int (pts0, GST_BUFFER_PTS (buf)); - fail_unless_equals_int (dur, GST_BUFFER_DURATION (buf)); + fail_unless_equals_uint64 (pts0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); fail_unless (GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DISCONT)); gst_buffer_unref (buf); @@ -1121,14 +1177,14 @@ GST_START_TEST (audiodecoder_plc_on_gap_event_with_delay) GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DISCONT); gst_harness_push (h, buf); buf = gst_harness_pull (h); - fail_unless_equals_int (pts0, GST_BUFFER_PTS (buf)); - fail_unless_equals_int (dur, GST_BUFFER_DURATION (buf)); + fail_unless_equals_uint64 (pts0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); fail_unless (!GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DISCONT)); gst_buffer_unref (buf); buf = gst_harness_pull (h); - fail_unless_equals_int (pts1, GST_BUFFER_PTS (buf)); - fail_unless_equals_int (dur, GST_BUFFER_DURATION (buf)); + fail_unless_equals_uint64 (pts1, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); fail_unless (!GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_DISCONT)); gst_buffer_unref (buf); gst_harness_teardown (h); @@ -1136,6 +1192,150 @@ GST_START_TEST (audiodecoder_plc_on_gap_event_with_delay) GST_END_TEST; +GST_START_TEST (audiodecoder_dtx) +{ + guint pkt = 0; + GstClockTime pts, dur; + GstHarness *h = setup_audiodecodertester (NULL, NULL); + gst_audio_decoder_set_plc_aware (GST_AUDIO_DECODER (h->element), TRUE); + gst_audio_decoder_set_plc (GST_AUDIO_DECODER (h->element), TRUE); + + /* start/stop DTX 3 times */ + for (gint i = 0; i < 3; i++) { + GstBuffer *buf; + + buf = create_dtx_buffer (pkt++); + pts = GST_BUFFER_PTS (buf); + dur = GST_BUFFER_DURATION (buf); + fail_unless_equals_int64 (GST_FLOW_OK, gst_harness_push (h, buf)); + + /* the DTX packet goes through */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); + fail_unless_silent_buffer (buf); + + /* expect some silent buffers from the DTX thread */ + fail_unless_silent_buffers (h, 2, pts); + /* advance two packets */ + pkt += 2; + pts += 2 * dur; + + /* and audio buffer arrives, the decoder should stop DTX */ + buf = create_test_buffer (pkt++); + pts = GST_BUFFER_PTS (buf); + dur = GST_BUFFER_DURATION (buf); + fail_unless_equals_int64 (GST_FLOW_OK, gst_harness_push (h, buf)); + + /* that audio buffer goes through */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + /* the decoder does not produce more packets */ + fail_unless (gst_harness_try_pull (h) == NULL); + } + + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (audiodecoder_lost_pkt_stops_dtx) +{ + GstClockTime pts, dur; + GstClockTime lost_pts, lost_dur; + GstBuffer *buf; + GstHarness *h = setup_audiodecodertester (NULL, NULL); + + gst_audio_decoder_set_plc_aware (GST_AUDIO_DECODER (h->element), TRUE); + gst_audio_decoder_set_plc (GST_AUDIO_DECODER (h->element), TRUE); + + /* start with DTX in */ + buf = create_dtx_buffer (0); + pts = GST_BUFFER_PTS (buf); + dur = GST_BUFFER_DURATION (buf); + fail_unless_equals_int64 (GST_FLOW_OK, gst_harness_push (h, buf)); + + /* the DTX packet goes through */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); + fail_unless_silent_buffer (buf); + + /* lets receive 5 silent buffers */ + fail_unless_silent_buffers (h, 5, pts); + + /* buffer 4 is lost, dtx should be stopped now */ + buf = create_test_buffer (4); + lost_pts = GST_BUFFER_PTS (buf); + lost_dur = GST_BUFFER_DURATION (buf); + gst_harness_push_event (h, gst_event_new_gap (lost_pts, lost_dur)); + gst_buffer_unref (buf); + + /* audio buffer 5 arrives */ + buf = create_test_buffer (5); + pts = GST_BUFFER_PTS (buf); + dur = GST_BUFFER_DURATION (buf); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + + /* expect a conceal buffer 4 is generated for the lost one */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (lost_pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (lost_dur, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + /* and expect buffer 5 */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (pts, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (dur, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + gst_harness_teardown (h); +} + +GST_END_TEST; + +static GstBuffer * +audiodecoder_dtx_stress_prepare_buffer (G_GNUC_UNUSED GstHarness * h, + gpointer data) +{ + gint *i = data; + GstBuffer *buf = create_test_buffer (*i); + *i = *i + 1; + return buf; +} + +GST_START_TEST (audiodecoder_dtx_stress_push) +{ + gint i = 0; + GstCaps *caps; + GstSegment segment; + GstHarnessThread *push_t; + GstHarness *h = setup_audiodecodertester (NULL, NULL); + + gst_audio_decoder_set_plc_aware (GST_AUDIO_DECODER (h->element), TRUE); + gst_audio_decoder_set_plc (GST_AUDIO_DECODER (h->element), TRUE); + gst_harness_set_drop_buffers (h, TRUE); + gst_harness_use_systemclock (h); + + caps = gst_caps_new_simple ("audio/x-test-custom", + "channels", G_TYPE_INT, 2, "rate", G_TYPE_INT, 44100, NULL); + gst_segment_init (&segment, GST_FORMAT_TIME); + + push_t = gst_harness_stress_push_buffer_with_cb_start (h, caps, &segment, + audiodecoder_dtx_stress_prepare_buffer, &i, NULL); + + g_usleep (1 * G_USEC_PER_SEC); + + gst_harness_stress_thread_stop (push_t); + gst_caps_unref (caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + static Suite * gst_audiodecoder_suite (void) { @@ -1166,6 +1366,11 @@ gst_audiodecoder_suite (void) tcase_add_test (tc, audiodecoder_plc_on_gap_event); tcase_add_test (tc, audiodecoder_plc_on_gap_event_with_delay); + suite_add_tcase (s, tc = tcase_create ("dtx")); + tcase_add_test (tc, audiodecoder_dtx); + tcase_add_test (tc, audiodecoder_lost_pkt_stops_dtx); + tcase_add_test (tc, audiodecoder_dtx_stress_push); + return s; } diff --git a/subprojects/gst-plugins-base/tests/check/libs/audioencoder.c b/subprojects/gst-plugins-base/tests/check/libs/audioencoder.c index bcaf8d98b8c..93ece17cbbf 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/audioencoder.c +++ b/subprojects/gst-plugins-base/tests/check/libs/audioencoder.c @@ -395,6 +395,46 @@ GST_START_TEST (audioencoder_events_before_eos) GST_END_TEST; +GST_START_TEST (audioencoder_headers_timestamp) +{ + GstHarness *h = setup_audioencodertester (); + GList *headers = NULL; + GstBuffer *buf; + + /* create and add headers */ + headers = g_list_append (headers, gst_buffer_new ()); + headers = g_list_append (headers, gst_buffer_new ()); + gst_audio_encoder_set_headers (GST_AUDIO_ENCODER (h->element), headers); + + /* push buffer with timestamp 10 seconds*/ + fail_unless (gst_harness_push (h, create_test_buffer (10)) == GST_FLOW_OK); + + /* first header */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (10 * GST_SECOND, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (10 * GST_SECOND, GST_BUFFER_DTS (buf)); + fail_unless_equals_uint64 (0, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + /* second header */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (10 * GST_SECOND, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (10 * GST_SECOND, GST_BUFFER_DTS (buf)); + fail_unless_equals_uint64 (0, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + /* buffer */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (10 * GST_SECOND, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (10 * GST_SECOND, GST_BUFFER_DTS (buf)); + fail_unless_equals_uint64 (1 * GST_SECOND, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + gst_harness_teardown (h); +} + +GST_END_TEST; + static Suite * gst_audioencoder_suite (void) { @@ -408,6 +448,8 @@ gst_audioencoder_suite (void) tcase_add_test (tc, audioencoder_events_before_eos); tcase_add_test (tc, audioencoder_flush_events); + tcase_add_test (tc, audioencoder_headers_timestamp); + return s; } diff --git a/subprojects/gst-plugins-base/tests/check/libs/rtp.c b/subprojects/gst-plugins-base/tests/check/libs/rtp.c index 1fd25f3d6b2..38b01ba237b 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/rtp.c +++ b/subprojects/gst-plugins-base/tests/check/libs/rtp.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #define RTP_HEADER_LEN 12 @@ -110,7 +111,9 @@ GST_START_TEST (test_rtp_buffer) /* check csrc bits */ fail_unless_equals_int (gst_rtp_buffer_get_csrc_count (&rtp), 0); - ASSERT_CRITICAL (gst_rtp_buffer_get_csrc (&rtp, 0)); + GST_FIXME + ("Unable to check CSRC. ASSERT_CRITICAL is broken. See MCU issue 21304"); + // ASSERT_CRITICAL (gst_rtp_buffer_get_csrc (&rtp, 0)); fail_unless_equals_int (data[0] & 0xf, 0); gst_rtp_buffer_unmap (&rtp); @@ -128,7 +131,9 @@ GST_START_TEST (test_rtp_buffer) gst_rtp_buffer_map (buf, GST_MAP_READWRITE, &rtp); fail_unless_equals_int (gst_rtp_buffer_get_csrc_count (&rtp), 3); - ASSERT_CRITICAL (gst_rtp_buffer_get_csrc (&rtp, 3)); + GST_FIXME + ("Unable to check CSRC. ASSERT_CRITICAL is broken. See MCU issue 21304"); + // ASSERT_CRITICAL (gst_rtp_buffer_get_csrc (&rtp, 3)); fail_unless_equals_int (data[0] & 0xf, 3); fail_unless_equals_int (gst_rtp_buffer_get_csrc (&rtp, 0), 0); fail_unless_equals_int (gst_rtp_buffer_get_csrc (&rtp, 1), 0); @@ -144,7 +149,9 @@ GST_START_TEST (test_rtp_buffer) fail_unless_equals_int (GST_READ_UINT32_BE (data + 1 * 4), 0xf7c1); gst_rtp_buffer_set_csrc (&rtp, 2, 0xf7c2); fail_unless_equals_int (GST_READ_UINT32_BE (data + 2 * 4), 0xf7c2); - ASSERT_CRITICAL (gst_rtp_buffer_set_csrc (&rtp, 3, 0xf123)); + GST_FIXME + ("Unable to check CSRC. ASSERT_CRITICAL is broken. See MCU issue 21304"); + // ASSERT_CRITICAL (gst_rtp_buffer_set_csrc (&rtp, 3, 0xf123)); gst_rtp_buffer_unmap (&rtp); gst_buffer_unmap (buf, &map); @@ -934,7 +941,7 @@ GST_START_TEST (test_rtcp_validate_with_padding) /* Compound packet with padding in the last packet. Padding is included in * the length of the last packet. */ guint8 rtcp_pkt[] = { - 0x80, 0xC9, 0x00, 0x07, /* Type RR, length = 7 */ + 0x80, 0xc9, 0x00, 0x07, /* Type RR, length = 7 */ 0x97, 0x6d, 0x21, 0x6a, 0x4d, 0x16, 0xaf, 0x14, 0x10, 0x1f, 0xd9, 0x91, @@ -965,7 +972,7 @@ GST_START_TEST (test_rtcp_validate_with_padding_wrong_padlength) /* Compound packet with padding in the last packet. Padding is included in * the length of the last packet. */ guint8 rtcp_pkt[] = { - 0x80, 0xC9, 0x00, 0x07, /* Type RR, length = 7 */ + 0x80, 0xc9, 0x00, 0x07, /* Type RR, length = 7 */ 0x97, 0x6d, 0x21, 0x6a, 0x4d, 0x16, 0xaf, 0x14, 0x10, 0x1f, 0xd9, 0x91, @@ -996,7 +1003,7 @@ GST_START_TEST (test_rtcp_validate_with_padding_excluded_from_length) /* Compound packet with padding in the last packet. Padding is not included * in the length. */ guint8 rtcp_pkt[] = { - 0x80, 0xC9, 0x00, 0x07, /* Type RR, length = 7 */ + 0x80, 0xc9, 0x00, 0x07, /* Type RR, length = 7 */ 0x97, 0x6d, 0x21, 0x6a, 0x4d, 0x16, 0xaf, 0x14, 0x10, 0x1f, 0xd9, 0x91, @@ -1027,7 +1034,7 @@ GST_START_TEST (test_rtcp_validate_with_padding_set_in_first_packet) /* Compound packet with padding in the last packet but with the pad bit set on first packet */ guint8 rtcp_pkt[] = { - 0xA0, 0xC9, 0x00, 0x07, /* P=1, Type RR, length = 7 */ + 0xa0, 0xc9, 0x00, 0x07, /* P=1, Type RR, length = 7 */ 0x97, 0x6d, 0x21, 0x6a, 0x4d, 0x16, 0xaf, 0x14, 0x10, 0x1f, 0xd9, 0x91, @@ -1092,6 +1099,118 @@ GST_START_TEST (test_rtcp_validate_reduced_with_padding) GST_END_TEST; +GST_START_TEST (test_rtcp_validate_reduced_with_invalid_padding) +{ + /* Reduced size packet with padding. */ + guint8 rtcp_pkt[] = { + 0xa0, 0xcd, 0x00, 0x01, /* P=1, PT=205(FB), length=1 (x4 = 4 bytes) */ + 0x00, 0x00, 0x00, 0x05 /* 5x padding bytes, more than total */ + }; + + fail_if (gst_rtcp_buffer_validate_data_reduced (rtcp_pkt, sizeof (rtcp_pkt))); +} + +GST_END_TEST; + +GST_START_TEST (test_rtcp_validate_reduced_with_invalid_padding_and_length) +{ + /* Reduced size packet with padding. */ + guint8 rtcp_pkt0[] = { + 0xa0, 0xcd, 0x00, 0x00, /* P=1, PT=205(FB), length=0 */ + }; + guint8 rtcp_pkt1[] = { + 0xa0, 0xcd, 0x00, 0x01, /* P=1, PT=205(FB), length=1 */ + }; + + fail_if (gst_rtcp_buffer_validate_data_reduced (rtcp_pkt0, + sizeof (rtcp_pkt0))); + fail_if (gst_rtcp_buffer_validate_data_reduced (rtcp_pkt1, + sizeof (rtcp_pkt1))); +} + +GST_END_TEST; + +GST_START_TEST (test_rtcp_validate_and_parse_chrome_twcc) +{ + GstBuffer *buffer; + GstRTCPPacket packet; + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + + guint8 rtcp_pkt[] = { + 0x8f, 0xcd, 0x00, 0x05, /* P=0, FMT=15, PT=205(FB), length=5 (x4 = 20 bytes) */ + 0xce, 0x0d, 0xc2, 0xb3, /* SSRC of packet sender */ + 0x53, 0xf6, 0x11, 0x3b, /* SSRC of media source */ + 0x00, 0x37, 0x00, 0x02, /* start of FCI */ + 0x00, 0xce, 0x40, 0x02, + 0x20, 0x02, 0x84, 0x50 /* no padding */ + }; + buffer = gst_buffer_new_wrapped_full (GST_MEMORY_FLAG_READONLY, + rtcp_pkt, sizeof (rtcp_pkt), 0, sizeof (rtcp_pkt), NULL, NULL); + + fail_unless (gst_rtcp_buffer_validate_reduced (buffer)); + + gst_rtcp_buffer_map (buffer, GST_MAP_READ, &rtcp); + + fail_unless (gst_rtcp_buffer_get_first_packet (&rtcp, &packet)); + + fail_unless_equals_int (GST_RTCP_TYPE_RTPFB, + gst_rtcp_packet_get_type (&packet)); + fail_unless_equals_int (GST_RTCP_RTPFB_TYPE_TWCC, + gst_rtcp_packet_fb_get_type (&packet)); + + fail_unless_equals_int (3, gst_rtcp_packet_fb_get_fci_length (&packet)); + fail_unless_equals_int (12, + gst_rtcp_packet_fb_get_fci_length_bytes (&packet)); + + gst_rtcp_buffer_unmap (&rtcp); + gst_buffer_unref (buffer); + +} + +GST_END_TEST; + +GST_START_TEST (test_rtcp_validate_and_parse_padded_chrome_twcc) +{ + GstBuffer *buffer; + GstRTCPPacket packet; + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + + guint8 rtcp_pkt[] = { + 0xaf, 0xcd, 0x00, 0x06, /* P=1, FMT=15, FTPT=205(FB), length=6 (x4 = 24 bytes) */ + 0xce, 0x0d, 0xc2, 0xb3, /* SSRC of packet sender */ + 0x53, 0xf6, 0x11, 0x3b, /* SSRC of media source */ + 0x00, 0x34, 0x00, 0x03, /* start of FCI */ + 0x00, 0xce, 0x3f, 0x01, + 0x20, 0x03, 0x94, 0x50, + 0x50, 0x00, 0x00, 0x03 /* 3x padding bytes */ + }; + + buffer = gst_buffer_new_wrapped_full (GST_MEMORY_FLAG_READONLY, + rtcp_pkt, sizeof (rtcp_pkt), 0, sizeof (rtcp_pkt), NULL, NULL); + + fail_unless (gst_rtcp_buffer_validate_reduced (buffer)); + + gst_rtcp_buffer_map (buffer, GST_MAP_READ, &rtcp); + + fail_unless (gst_rtcp_buffer_get_first_packet (&rtcp, &packet)); + + fail_unless_equals_int (GST_RTCP_TYPE_RTPFB, + gst_rtcp_packet_get_type (&packet)); + fail_unless_equals_int (GST_RTCP_RTPFB_TYPE_TWCC, + gst_rtcp_packet_fb_get_type (&packet)); + + /* This is actually "wrong", but the resolution for fci_length is in 32bits */ + fail_unless_equals_int (4, gst_rtcp_packet_fb_get_fci_length (&packet)); + /* hence the introduction of this API: */ + fail_unless_equals_int (13, + gst_rtcp_packet_fb_get_fci_length_bytes (&packet)); + + gst_rtcp_buffer_unmap (&rtcp); + gst_buffer_unref (buffer); +} + +GST_END_TEST; + GST_START_TEST (test_rtcp_buffer_profile_specific_extension) { GstBuffer *buf; @@ -2323,6 +2442,106 @@ GST_START_TEST (test_rtp_buffer_remove_extension_data) GST_END_TEST; +typedef struct +{ + const gchar *name; + guint x; + guint y; + guint w; + guint h; +} ROI; + +static void +_add_roi (GstBuffer * buf, ROI * roi) +{ + gst_buffer_add_video_region_of_interest_meta (buf, + roi->name, roi->x, roi->y, roi->w, roi->h); +} + +static void +_meta_to_roi (GstVideoRegionOfInterestMeta * meta, ROI * roi) +{ + roi->x = meta->x; + roi->y = meta->y; + roi->w = meta->w; + roi->h = meta->h; +} + +static gboolean +_foreach_meta_func (G_GNUC_UNUSED GstBuffer * buf, GstMeta ** meta, + gpointer user_data) +{ + GArray *array = user_data; + const GstMetaInfo *info = (*meta)->info; + + if (info->api == GST_VIDEO_REGION_OF_INTEREST_META_API_TYPE) { + ROI roi; + _meta_to_roi ((GstVideoRegionOfInterestMeta *) * meta, &roi); + g_array_append_val (array, roi); + } + + return TRUE; +} + +static GArray * +_get_rois (GstBuffer * buf) +{ + GArray *array = g_array_new (FALSE, FALSE, sizeof (ROI)); + gst_buffer_foreach_meta (buf, _foreach_meta_func, array); + return array; +} + +GST_START_TEST (test_rtp_buffer_roi_meta_as_ext_hdr) +{ + GstBuffer *buf0 = gst_buffer_new (); + GstBuffer *buf1 = gst_buffer_new (); + GstBuffer *rtpbuf = gst_rtp_buffer_new_allocate (10, 0, 0); + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + GArray *result_rois; + guint i; + ROI expected_rois[] = { + {"test0", 10, 10, 10, 10} + , + {"test1", 20, 20, 20, 20} + , + {"test2", 30, 30, 30, 30} + , + }; + + /* add ROI meta to buf0 */ + for (i = 0; i < G_N_ELEMENTS (expected_rois); i++) + _add_roi (buf0, &expected_rois[i]); + + /* ROI meta -> RTP header extension */ + gst_rtp_buffer_map (rtpbuf, GST_MAP_READWRITE, &rtp); + gst_rtp_buffer_video_roi_meta_to_one_byte_ext (&rtp, buf0, 10); + gst_rtp_buffer_unmap (&rtp); + + /* RTP header extension -> ROI meta */ + gst_rtp_buffer_map (rtpbuf, GST_MAP_READWRITE, &rtp); + gst_rtp_buffer_video_roi_meta_from_one_byte_ext (&rtp, buf1, 10); + gst_rtp_buffer_unmap (&rtp); + + /* verify the meta survived the conversion */ + result_rois = _get_rois (buf1); + fail_unless_equals_int (result_rois->len, G_N_ELEMENTS (expected_rois)); + for (i = 0; i < G_N_ELEMENTS (expected_rois); i++) { + ROI *exp = &expected_rois[i]; + ROI *act = &g_array_index (result_rois, ROI, i); + fail_unless_equals_int (exp->x, act->x); + fail_unless_equals_int (exp->y, act->y); + fail_unless_equals_int (exp->w, act->w); + fail_unless_equals_int (exp->h, act->h); + } + g_array_unref (result_rois); + + gst_buffer_unref (rtpbuf); + gst_buffer_unref (buf0); + gst_buffer_unref (buf1); +} + +GST_END_TEST; + static Suite * rtp_suite (void) { @@ -2348,6 +2567,12 @@ rtp_suite (void) test_rtcp_validate_with_padding_set_in_first_packet); tcase_add_test (tc_chain, test_rtcp_validate_reduced_without_padding); tcase_add_test (tc_chain, test_rtcp_validate_reduced_with_padding); + tcase_add_test (tc_chain, test_rtcp_validate_reduced_with_invalid_padding); + tcase_add_test (tc_chain, + test_rtcp_validate_reduced_with_invalid_padding_and_length); + tcase_add_test (tc_chain, test_rtcp_validate_and_parse_chrome_twcc); + tcase_add_test (tc_chain, test_rtcp_validate_and_parse_padded_chrome_twcc); + tcase_add_test (tc_chain, test_rtcp_buffer_profile_specific_extension); tcase_add_test (tc_chain, test_rtcp_buffer_app); tcase_add_test (tc_chain, test_rtcp_buffer_xr); @@ -2380,6 +2605,8 @@ rtp_suite (void) tcase_add_test (tc_chain, test_rtp_buffer_remove_extension_data); tcase_add_test (tc_chain, test_rtp_buffer_set_extension_data_shrink_data); + tcase_add_test (tc_chain, test_rtp_buffer_roi_meta_as_ext_hdr); + return s; } diff --git a/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c b/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c index 29e9098663e..f802ebbccd7 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c +++ b/subprojects/gst-plugins-base/tests/check/libs/rtpbasedepayload.c @@ -186,7 +186,6 @@ gst_rtp_dummy_depay_process (GstRTPBaseDepayload * depayload, GstBuffer * buf) break; case GST_RTP_DUMMY_USE_PUSH_LIST_FUNC:{ GstBufferList *blist = gst_buffer_list_new (); - gint i; gst_buffer_list_add (blist, outbuf); for (i = 0; i != self->num_buffers_in_blist - 1; ++i) { gst_buffer_list_add (blist, gst_buffer_copy (outbuf)); @@ -406,12 +405,15 @@ rtp_buffer_set_valist (GstBuffer * buf, const gchar * field, va_list var_args, while (field) { if (!g_strcmp0 (field, "pts")) { + fail_unless (!mapped); GstClockTime pts = va_arg (var_args, GstClockTime); GST_BUFFER_PTS (buf) = pts; } else if (!g_strcmp0 (field, "offset")) { + fail_unless (!mapped); guint64 offset = va_arg (var_args, guint64); GST_BUFFER_OFFSET (buf) = offset; } else if (!g_strcmp0 (field, "discont")) { + fail_unless (!mapped); gboolean discont = va_arg (var_args, gboolean); if (discont) { GST_BUFFER_FLAG_SET (buf, GST_BUFFER_FLAG_DISCONT); @@ -1102,10 +1104,10 @@ GST_START_TEST (rtp_base_depayload_seq_discont_test) "rtptime", G_GUINT64_CONSTANT (0x1234), "seq", 1, NULL); push_rtp_buffer (state, - "extra-ref", TRUE, "pts", 2 * GST_SECOND, "rtptime", G_GUINT64_CONSTANT (0x1234) + DEFAULT_CLOCK_RATE / 2, - "seq", 33333, NULL); + "seq", 33333, + "extra-ref", TRUE, NULL); set_state (state, GST_STATE_NULL); @@ -1679,7 +1681,7 @@ GST_START_TEST (rtp_base_depayload_two_byte_hdr_ext) GST_END_TEST; static GstRTPHeaderExtension * -request_extension (GstRTPBaseDepayload * depayload, guint ext_id, +request_extension (G_GNUC_UNUSED GstRTPBaseDepayload * depayload, guint ext_id, const gchar * ext_uri, gpointer user_data) { GstRTPHeaderExtension *ext = user_data; @@ -1815,8 +1817,9 @@ GST_START_TEST (rtp_base_depayload_multiple_exts) GST_END_TEST; static GstRTPHeaderExtension * -request_extension_ignored (GstRTPBaseDepayload * depayload, guint ext_id, - const gchar * ext_uri, gpointer user_data) +request_extension_ignored (G_GNUC_UNUSED GstRTPBaseDepayload * depayload, + G_GNUC_UNUSED guint ext_id, G_GNUC_UNUSED const gchar * ext_uri, + gpointer user_data) { guint *request_counter = user_data; @@ -2084,6 +2087,35 @@ GST_START_TEST (rtp_base_depayload_hdr_ext_aggregate_flush) GST_END_TEST; +GST_START_TEST (rtp_base_depayload_rtptime_buffer_offset) +{ + State *state; + state = create_depayloader ("application/x-rtp", NULL); + set_state (state, GST_STATE_PLAYING); + gint buf_idx = 0; + + /* buffer offset should start at 0 */ + push_rtp_buffer (state, "pts", buf_idx * GST_SECOND, + "rtptime", 1000, "seq", 0x4242 + buf_idx, NULL); + validate_buffer (buf_idx++, "offset", 0, NULL); + + /* buffer offset should be the delta between this rtptime and first + * rtptime */ + push_rtp_buffer (state, "pts", buf_idx * GST_SECOND, + "rtptime", 1500, "seq", 0x4242 + buf_idx, NULL); + validate_buffer (buf_idx++, "offset", 500, NULL); + + /* buffer offset already set so should not be modified */ + push_rtp_buffer (state, "pts", buf_idx * GST_SECOND, "offset", 5678, + "rtptime", 2000, "seq", 0x4242 + buf_idx, NULL); + validate_buffer (buf_idx++, "offset", 5678, NULL); + + set_state (state, GST_STATE_NULL); + destroy_depayloader (state); +} + +GST_END_TEST; + static Suite * rtp_basepayloading_suite (void) { @@ -2127,11 +2159,15 @@ rtp_basepayloading_suite (void) tcase_add_test (tc_chain, rtp_base_depayload_clear_extensions); tcase_add_test (tc_chain, rtp_base_depayload_multiple_exts); tcase_add_test (tc_chain, rtp_base_depayload_caps_request_ignored); + tcase_add_test (tc_chain, rtp_base_depayload_hdr_ext_caps_change); tcase_add_test (tc_chain, rtp_base_depayload_hdr_ext_aggregate); tcase_add_test (tc_chain, rtp_base_depayload_hdr_ext_aggregate_drop); tcase_add_test (tc_chain, rtp_base_depayload_hdr_ext_aggregate_delayed); tcase_add_test (tc_chain, rtp_base_depayload_hdr_ext_aggregate_flush); + + tcase_add_test (tc_chain, rtp_base_depayload_rtptime_buffer_offset); + return s; } diff --git a/subprojects/gst-plugins-base/tests/check/libs/rtpbasepayload.c b/subprojects/gst-plugins-base/tests/check/libs/rtpbasepayload.c index a7cfa46ffd2..9883f624285 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/rtpbasepayload.c +++ b/subprojects/gst-plugins-base/tests/check/libs/rtpbasepayload.c @@ -2146,7 +2146,7 @@ GST_STATIC_PAD_TEMPLATE ("sink", DUMMY_HDR_EXT_URI)); static GstRTPHeaderExtension * -request_extension (GstRTPBasePayload * depayload, guint ext_id, +request_extension (G_GNUC_UNUSED GstRTPBasePayload * payload, guint ext_id, const gchar * ext_uri, gpointer user_data) { GstRTPHeaderExtension *ext = user_data; @@ -2199,8 +2199,9 @@ GST_START_TEST (rtp_base_payload_caps_request) GST_END_TEST; static GstRTPHeaderExtension * -request_extension_ignored (GstRTPBasePayload * depayload, guint ext_id, - const gchar * ext_uri, gpointer user_data) +request_extension_ignored (G_GNUC_UNUSED GstRTPBasePayload * payload, + G_GNUC_UNUSED guint ext_id, G_GNUC_UNUSED const gchar * ext_uri, + gpointer user_data) { guint *request_counter = user_data; diff --git a/subprojects/gst-plugins-base/tests/check/libs/rtpdummyhdrextimpl.c b/subprojects/gst-plugins-base/tests/check/libs/rtpdummyhdrextimpl.c index 9910d9d61d5..fda8efde6cd 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/rtpdummyhdrextimpl.c +++ b/subprojects/gst-plugins-base/tests/check/libs/rtpdummyhdrextimpl.c @@ -160,8 +160,8 @@ gst_rtp_dummy_hdr_ext_get_supported_flags (GstRTPHeaderExtension * ext) } static gsize -gst_rtp_dummy_hdr_ext_get_max_size (GstRTPHeaderExtension * ext, - const GstBuffer * input_meta) +gst_rtp_dummy_hdr_ext_get_max_size (G_GNUC_UNUSED GstRTPHeaderExtension * ext, + G_GNUC_UNUSED const GstBuffer * input_meta) { GstRTPDummyHdrExt *dummy = GST_RTP_DUMMY_HDR_EXT (ext); @@ -172,8 +172,9 @@ gst_rtp_dummy_hdr_ext_get_max_size (GstRTPHeaderExtension * ext, static gssize gst_rtp_dummy_hdr_ext_write (GstRTPHeaderExtension * ext, - const GstBuffer * input_meta, GstRTPHeaderExtensionFlags write_flags, - GstBuffer * output, guint8 * data, gsize size) + G_GNUC_UNUSED const GstBuffer * input_meta, + G_GNUC_UNUSED GstRTPHeaderExtensionFlags write_flags, + G_GNUC_UNUSED GstBuffer * output, guint8 * data, gsize size) { GstRTPDummyHdrExt *dummy = GST_RTP_DUMMY_HDR_EXT (ext); @@ -188,8 +189,8 @@ gst_rtp_dummy_hdr_ext_write (GstRTPHeaderExtension * ext, static gboolean gst_rtp_dummy_hdr_ext_read (GstRTPHeaderExtension * ext, - GstRTPHeaderExtensionFlags read_flags, const guint8 * data, - gsize size, GstBuffer * buffer) + G_GNUC_UNUSED GstRTPHeaderExtensionFlags read_flags, const guint8 * data, + G_GNUC_UNUSED gsize size, G_GNUC_UNUSED GstBuffer * buffer) { GstRTPDummyHdrExt *dummy = GST_RTP_DUMMY_HDR_EXT (ext); @@ -218,7 +219,8 @@ gst_rtp_dummy_hdr_ext_set_caps_from_attributes (GstRTPHeaderExtension * ext, static gboolean gst_rtp_dummy_hdr_ext_set_attributes (GstRTPHeaderExtension * ext, - GstRTPHeaderExtensionDirection direction, const gchar * attributes) + G_GNUC_UNUSED GstRTPHeaderExtensionDirection direction, + const gchar * attributes) { GstRTPDummyHdrExt *dummy = GST_RTP_DUMMY_HDR_EXT (ext); diff --git a/subprojects/gst-plugins-base/tests/check/libs/videodecoder.c b/subprojects/gst-plugins-base/tests/check/libs/videodecoder.c index c995321e20d..4008a0a41a6 100644 --- a/subprojects/gst-plugins-base/tests/check/libs/videodecoder.c +++ b/subprojects/gst-plugins-base/tests/check/libs/videodecoder.c @@ -1578,7 +1578,76 @@ GST_START_TEST (videodecoder_playback_invalid_ts_packetized_subframes) GST_END_TEST; +static void +test_videodecoder_property_detect_reordering (gboolean detect_reordering) +{ + GstSegment segment; + GstBuffer *buffer; + GstClockTime last_timestamp = GST_CLOCK_TIME_NONE; + guint i = 0; + + GstClockTime timestamps[] = { + 1000, + 2000, + 1500, + 2500, + 3000 + }; + + setup_videodecodertester (NULL, NULL); + + gst_pad_set_active (mysrcpad, TRUE); + g_object_set (dec, "detect-reordering", detect_reordering, NULL); + gst_element_set_state (dec, GST_STATE_PLAYING); + gst_pad_set_active (mysinkpad, TRUE); + + send_startup_events (); + + /* push a new segment */ + gst_segment_init (&segment, GST_FORMAT_TIME); + fail_unless (gst_pad_push_event (mysrcpad, gst_event_new_segment (&segment))); + + /* push a number of buffers with their corresponding timestamp */ + for (i = 0; i < G_N_ELEMENTS (timestamps); i++) { + buffer = create_test_buffer (i); + GST_BUFFER_PTS (buffer) = timestamps[i]; + fail_unless (gst_pad_push (mysrcpad, buffer) == GST_FLOW_OK); + } + for (i = 0; i < g_list_length (buffers); i++) { + buffer = g_list_nth_data (buffers, i); + + if (detect_reordering) { + /* when reordering we expect PTS to be continuous */ + fail_unless (GST_CLOCK_DIFF (last_timestamp, + GST_BUFFER_PTS (buffer)) >= 0); + last_timestamp = GST_BUFFER_PTS (buffer); + + } else { + /* we expect PTS not to be modified */ + fail_unless_equals_int (GST_BUFFER_PTS (buffer), timestamps[i]); + } + } + + g_list_free_full (buffers, (GDestroyNotify) gst_buffer_unref); + buffers = NULL; + + cleanup_videodecodertest (); +} + +GST_START_TEST (videodecoder_property_detect_reordering_enabled) +{ + test_videodecoder_property_detect_reordering (TRUE); +} + +GST_END_TEST; + +GST_START_TEST (videodecoder_property_detect_reordering_disabled) +{ + test_videodecoder_property_detect_reordering (FALSE); +} + +GST_END_TEST; static Suite * gst_videodecoder_suite (void) @@ -1616,6 +1685,9 @@ gst_videodecoder_suite (void) tcase_add_test (tc, videodecoder_playback_invalid_ts_packetized); tcase_add_test (tc, videodecoder_playback_invalid_ts_packetized_subframes); + tcase_add_test (tc, videodecoder_property_detect_reordering_enabled); + tcase_add_test (tc, videodecoder_property_detect_reordering_disabled); + return s; } diff --git a/subprojects/gst-plugins-good/docs/gst_plugins_cache.json b/subprojects/gst-plugins-good/docs/gst_plugins_cache.json index bf5ad6657b6..e76bbd5264d 100644 --- a/subprojects/gst-plugins-good/docs/gst_plugins_cache.json +++ b/subprojects/gst-plugins-good/docs/gst_plugins_cache.json @@ -19339,6 +19339,20 @@ "readable": false, "type": "GstStructure", "writable": true + }, + "stuffing-kbps": { + "blurb": "Use RTX to send stuffing packets to produce a near constant rate of total packets (media and stuffing) (0 = disabled)", + "conditionally-available": false, + "construct": true, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "2147483647", + "min": "0", + "mutable": "null", + "readable": true, + "type": "gint", + "writable": true } }, "rank": "none", @@ -19398,6 +19412,34 @@ "type": "GstStructure", "writable": true }, + "max-bucket-size": { + "blurb": "The size of the token bucket, related to burstiness resilience (-1 = unlimited)", + "conditionally-available": false, + "construct": true, + "construct-only": false, + "controllable": false, + "default": "-1", + "max": "2147483647", + "min": "-1", + "mutable": "null", + "readable": true, + "type": "gint", + "writable": true + }, + "max-kbps": { + "blurb": "The maximum number of kilobits of RTX packets to allow (-1 = unlimited)", + "conditionally-available": false, + "construct": true, + "construct-only": false, + "controllable": false, + "default": "-1", + "max": "2147483647", + "min": "-1", + "mutable": "null", + "readable": true, + "type": "gint", + "writable": true + }, "max-size-packets": { "blurb": "Amount of packets to queue (0 = unlimited)", "conditionally-available": false, @@ -19475,6 +19517,20 @@ "readable": false, "type": "GstStructure", "writable": true + }, + "stuffing-kbps": { + "blurb": "Use RTX to send stuffing packets to produce a near constant rate of total packets (media and stuffing) (0 = disabled)", + "conditionally-available": false, + "construct": true, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "2147483647", + "min": "0", + "mutable": "null", + "readable": true, + "type": "gint", + "writable": true } }, "rank": "none", diff --git a/subprojects/gst-plugins-good/ext/speex/gstspeexdec.c b/subprojects/gst-plugins-good/ext/speex/gstspeexdec.c index eeb7af7e8b3..6d1d2d103ba 100644 --- a/subprojects/gst-plugins-good/ext/speex/gstspeexdec.c +++ b/subprojects/gst-plugins-good/ext/speex/gstspeexdec.c @@ -51,6 +51,8 @@ GST_DEBUG_CATEGORY_STATIC (speexdec_debug); #define DEFAULT_ENH TRUE +#define GST_FLOW_NO_HEADER GST_FLOW_CUSTOM_ERROR + enum { ARG_0, @@ -195,6 +197,7 @@ gst_speex_dec_stop (GstAudioDecoder * dec) static GstFlowReturn gst_speex_dec_parse_header (GstSpeexDec * dec, GstBuffer * buf) { + SpeexHeader *header; GstMapInfo map; GstAudioInfo info; static const GstAudioChannelPosition chan_pos[2][2] = { @@ -205,12 +208,18 @@ gst_speex_dec_parse_header (GstSpeexDec * dec, GstBuffer * buf) /* get the header */ gst_buffer_map (buf, &map, GST_MAP_READ); - dec->header = speex_packet_to_header ((gchar *) map.data, map.size); + header = speex_packet_to_header ((gchar *) map.data, map.size); gst_buffer_unmap (buf, &map); - if (!dec->header) + if (!header) goto no_header; + if (dec->header) { + GST_DEBUG_OBJECT (dec, "Replacing speex-header, resetting state"); + gst_speex_dec_reset (dec); + } + dec->header = header; + if (dec->header->mode >= SPEEX_NB_MODES || dec->header->mode < 0) goto mode_too_old; @@ -254,9 +263,8 @@ gst_speex_dec_parse_header (GstSpeexDec * dec, GstBuffer * buf) /* ERRORS */ no_header: { - GST_ELEMENT_ERROR (GST_ELEMENT (dec), STREAM, DECODE, - (NULL), ("couldn't read header")); - return GST_FLOW_ERROR; + GST_INFO_OBJECT (dec, "couldn't read header"); + return GST_FLOW_NO_HEADER; } mode_too_old: { @@ -290,7 +298,7 @@ gst_speex_dec_parse_comments (GstSpeexDec * dec, GstBuffer * buf) if (!list) { GST_WARNING_OBJECT (dec, "couldn't decode comments"); - list = gst_tag_list_new_empty (); + return GST_FLOW_NO_HEADER; } if (encoder) { @@ -473,6 +481,9 @@ memcmp_buffers (GstBuffer * buf1, GstBuffer * buf2) gsize size1, size2; gboolean res; + if (buf1 == NULL || buf2 == NULL) + return FALSE; + size1 = gst_buffer_get_size (buf1); size2 = gst_buffer_get_size (buf2); @@ -489,8 +500,9 @@ memcmp_buffers (GstBuffer * buf1, GstBuffer * buf2) static GstFlowReturn gst_speex_dec_handle_frame (GstAudioDecoder * bdec, GstBuffer * buf) { - GstFlowReturn res; + GstFlowReturn res = GST_FLOW_OK; GstSpeexDec *dec; + gboolean header_packet = FALSE; /* no fancy draining */ if (G_UNLIKELY (!buf)) @@ -498,42 +510,53 @@ gst_speex_dec_handle_frame (GstAudioDecoder * bdec, GstBuffer * buf) dec = GST_SPEEX_DEC (bdec); - /* If we have the streamheader and vorbiscomment from the caps already - * ignore them here */ - if (dec->streamheader && dec->vorbiscomment) { - if (memcmp_buffers (dec->streamheader, buf)) { - GST_DEBUG_OBJECT (dec, "found streamheader"); - gst_audio_decoder_finish_frame (bdec, NULL, 1); - res = GST_FLOW_OK; - } else if (memcmp_buffers (dec->vorbiscomment, buf)) { - GST_DEBUG_OBJECT (dec, "found vorbiscomments"); - gst_audio_decoder_finish_frame (bdec, NULL, 1); - res = GST_FLOW_OK; - } else { - res = gst_speex_dec_parse_data (dec, buf); - } - } else { - /* Otherwise fall back to packet counting and assume that the - * first two packets are the headers. */ - switch (dec->packetno) { - case 0: - GST_DEBUG_OBJECT (dec, "counted streamheader"); + switch (dec->packetno) { + case 0: + GST_DEBUG_OBJECT (dec, "expecting streamheader"); + header_packet = TRUE; + if (!memcmp_buffers (dec->streamheader, buf)) { res = gst_speex_dec_parse_header (dec, buf); - gst_audio_decoder_finish_frame (bdec, NULL, 1); - break; - case 1: - GST_DEBUG_OBJECT (dec, "counted vorbiscomments"); + if (res == GST_FLOW_NO_HEADER) { + header_packet = FALSE; + GST_INFO_OBJECT (dec, "No streamheader in first buffer"); + if (dec->streamheader == NULL) { + GST_ERROR_OBJECT (dec, "Can't proceed without a header"); + return GST_FLOW_ERROR; + } else { + GST_INFO_OBJECT (dec, "Using streamheader from caps"); + } + } + /* we prefer "inband" streamheaders to the ones in caps */ + if (res == GST_FLOW_OK) { + GST_DEBUG_OBJECT (dec, "found streamheader"); + gst_buffer_replace (&dec->streamheader, buf); + } + } + break; + case 1: + GST_DEBUG_OBJECT (dec, "expecting vorbiscomment"); + header_packet = TRUE; + if (!memcmp_buffers (dec->vorbiscomment, buf)) { res = gst_speex_dec_parse_comments (dec, buf); - gst_audio_decoder_finish_frame (bdec, NULL, 1); - break; - default: - { - res = gst_speex_dec_parse_data (dec, buf); - break; + if (res == GST_FLOW_NO_HEADER) { + header_packet = FALSE; + GST_INFO_OBJECT (dec, "No vorbisheader in second buffer"); + } + if (res == GST_FLOW_OK) { + GST_DEBUG_OBJECT (dec, "found vorbiscomment"); + gst_buffer_replace (&dec->vorbiscomment, buf); + } } - } + break; + default: + break; } + if (header_packet) + gst_audio_decoder_finish_frame (bdec, NULL, 1); + else + res = gst_speex_dec_parse_data (dec, buf); + dec->packetno++; return res; diff --git a/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c b/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c index 3eac649a398..abdd6ecfd9f 100644 --- a/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c +++ b/subprojects/gst-plugins-good/ext/speex/gstspeexenc.c @@ -314,7 +314,10 @@ gst_speex_enc_set_format (GstAudioEncoder * benc, GstAudioInfo * info) /* feedback to base class */ gst_audio_encoder_set_latency (benc, gst_speex_enc_get_latency (enc), gst_speex_enc_get_latency (enc)); - gst_audio_encoder_set_lookahead (benc, enc->lookahead); + + /* FIXME: awaiting a better solution, as lookahead does too much + wrong with both timestamp and duration. */ + //gst_audio_encoder_set_lookahead (benc, enc->lookahead); if (enc->nframes == 0) { /* as many frames as available input allows */ @@ -537,8 +540,8 @@ gst_speex_enc_encode (GstSpeexEnc * enc, GstBuffer * buf) gsize bsize, size; GstBuffer *outbuf; GstFlowReturn ret = GST_FLOW_OK; - GstSegment *segment; - GstClockTime duration; + //GstSegment *segment; + //GstClockTime duration; if (G_LIKELY (buf)) { gst_buffer_map (buf, &map, GST_MAP_READ); @@ -548,6 +551,7 @@ gst_speex_enc_encode (GstSpeexEnc * enc, GstBuffer * buf) if (G_UNLIKELY (bsize % bytes)) { GST_DEBUG_OBJECT (enc, "draining; adding silence samples"); +#if 0 /* If encoding part of a frame, and we have no set stop time on * the output segment, we update the segment stop time to reflect * the last sample. This will let oggmux set the last page's @@ -556,7 +560,8 @@ gst_speex_enc_encode (GstSpeexEnc * enc, GstBuffer * buf) segment = &GST_AUDIO_ENCODER_OUTPUT_SEGMENT (enc); GST_DEBUG_OBJECT (enc, "existing output segment %" GST_SEGMENT_FORMAT, segment); - if (!GST_CLOCK_TIME_IS_VALID (segment->stop)) { + if (!GST_CLOCK_TIME_IS_VALID (segment->stop) && + gst_pad_has_current_caps (GST_AUDIO_ENCODER_SRC_PAD (enc))) { int input_samples = bsize / (enc->channels * 2); GST_DEBUG_OBJECT (enc, "No stop time and partial frame, updating segment"); @@ -569,6 +574,7 @@ gst_speex_enc_encode (GstSpeexEnc * enc, GstBuffer * buf) gst_pad_push_event (GST_AUDIO_ENCODER_SRC_PAD (enc), gst_event_new_segment (segment)); } +#endif size = ((bsize / bytes) + 1) * bytes; data0 = data = g_malloc0 (size); diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvp8enc.c b/subprojects/gst-plugins-good/ext/vpx/gstvp8enc.c index 5222c970a1f..0a08ec20830 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvp8enc.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvp8enc.c @@ -98,6 +98,7 @@ gst_vp8_enc_user_data_free (GstVP8EncUserData * user_data) static vpx_codec_iface_t *gst_vp8_enc_get_algo (GstVPXEnc * enc); static gboolean gst_vp8_enc_enable_scaling (GstVPXEnc * enc); +static gboolean gst_vp8_enc_enable_tiles (GstVPXEnc * enc); static void gst_vp8_enc_set_image_format (GstVPXEnc * enc, vpx_image_t * image); static GstCaps *gst_vp8_enc_get_new_simple_caps (GstVPXEnc * enc); static void gst_vp8_enc_set_stream_info (GstVPXEnc * enc, GstCaps * caps, @@ -170,6 +171,7 @@ gst_vp8_enc_class_init (GstVP8EncClass * klass) vpx_encoder_class->get_algo = gst_vp8_enc_get_algo; vpx_encoder_class->enable_scaling = gst_vp8_enc_enable_scaling; + vpx_encoder_class->enable_tiles = gst_vp8_enc_enable_tiles; vpx_encoder_class->set_image_format = gst_vp8_enc_set_image_format; vpx_encoder_class->get_new_vpx_caps = gst_vp8_enc_get_new_simple_caps; vpx_encoder_class->set_stream_info = gst_vp8_enc_set_stream_info; @@ -218,6 +220,12 @@ gst_vp8_enc_enable_scaling (GstVPXEnc * enc) return TRUE; } +static gboolean +gst_vp8_enc_enable_tiles (GstVPXEnc * enc) +{ + return FALSE; +} + static void gst_vp8_enc_set_image_format (GstVPXEnc * enc, vpx_image_t * image) { diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvp9dec.c b/subprojects/gst-plugins-good/ext/vpx/gstvp9dec.c index 3819e602a3d..6828c57a852 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvp9dec.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvp9dec.c @@ -80,6 +80,7 @@ G_DEFINE_TYPE (GstVP9Dec, gst_vp9_dec, GST_TYPE_VPX_DEC); GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (vp9dec, "vp9dec", GST_RANK_PRIMARY, gst_vp9_dec_get_type (), vpx_element_init (plugin)); +#ifdef VPX_CODEC_CAP_HIGHBITDEPTH static GstCaps * gst_vp9_dec_get_src_caps (void) { @@ -90,6 +91,13 @@ gst_vp9_dec_get_src_caps (void) return gst_caps_from_string ((vpx_codec_get_caps (&vpx_codec_vp9_dx_algo) & VPX_CODEC_CAP_HIGHBITDEPTH) ? CAPS_HIGHBIT : CAPS_8BIT); } +#else +static GstCaps * +gst_vp9_dec_get_src_caps (void) +{ + return gst_caps_from_string (GST_VIDEO_CAPS_MAKE ("{ " GST_VP9_DEC_VIDEO_FORMATS_8BIT " }")); +} +#endif static void gst_vp9_dec_class_init (GstVP9DecClass * klass) diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c b/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c index e966e000c68..22608a17f5a 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.c @@ -105,6 +105,7 @@ GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (vp9enc, "vp9enc", GST_RANK_PRIMARY, static vpx_codec_iface_t *gst_vp9_enc_get_algo (GstVPXEnc * enc); static gboolean gst_vp9_enc_enable_scaling (GstVPXEnc * enc); +static gboolean gst_vp9_enc_enable_tiles (GstVPXEnc * enc); static void gst_vp9_enc_set_image_format (GstVPXEnc * enc, vpx_image_t * image); static GstCaps *gst_vp9_enc_get_new_simple_caps (GstVPXEnc * enc); static void gst_vp9_enc_set_stream_info (GstVPXEnc * enc, GstCaps * caps, @@ -233,6 +234,7 @@ gst_vp9_enc_class_init (GstVP9EncClass * klass) vpx_encoder_class->get_algo = gst_vp9_enc_get_algo; vpx_encoder_class->enable_scaling = gst_vp9_enc_enable_scaling; + vpx_encoder_class->enable_tiles = gst_vp9_enc_enable_tiles; vpx_encoder_class->set_image_format = gst_vp9_enc_set_image_format; vpx_encoder_class->get_new_vpx_caps = gst_vp9_enc_get_new_simple_caps; vpx_encoder_class->set_stream_info = gst_vp9_enc_set_stream_info; @@ -312,6 +314,7 @@ gst_vp9_enc_set_property (GObject * object, guint prop_id, case PROP_ROW_MT: gst_vp9_enc->row_mt = g_value_get_boolean (value); if (gst_vpx_enc->inited) { +#ifdef VPX_CTRL_VP9E_SET_ROW_MT status = vpx_codec_control (&gst_vpx_enc->encoder, VP9E_SET_ROW_MT, gst_vp9_enc->row_mt ? 1 : 0); @@ -319,6 +322,7 @@ gst_vp9_enc_set_property (GObject * object, guint prop_id, GST_WARNING_OBJECT (gst_vpx_enc, "Failed to set VP9E_SET_ROW_MT: %s", gst_vpx_error_name (status)); } +#endif } break; case PROP_AQ_MODE: @@ -464,6 +468,7 @@ gst_vp9_enc_configure_encoder (GstVPXEnc * encoder, GstVideoCodecState * state) GstVideoInfo *info = &state->info; vpx_codec_err_t status; +#if 0 status = vpx_codec_control (&encoder->encoder, VP9E_SET_COLOR_SPACE, gst_vp9_get_vpx_colorspace (encoder, &GST_VIDEO_INFO_COLORIMETRY (info), GST_VIDEO_INFO_FORMAT (info))); @@ -471,6 +476,7 @@ gst_vp9_enc_configure_encoder (GstVPXEnc * encoder, GstVideoCodecState * state) GST_WARNING_OBJECT (encoder, "Failed to set VP9E_SET_COLOR_SPACE: %s", gst_vpx_error_name (status)); } +#endif status = vpx_codec_control (&encoder->encoder, VP9E_SET_COLOR_RANGE, gst_vp9_get_vpx_color_range (&GST_VIDEO_INFO_COLORIMETRY (info))); @@ -494,6 +500,7 @@ gst_vp9_enc_configure_encoder (GstVPXEnc * encoder, GstVideoCodecState * state) GST_DEBUG_OBJECT (encoder, "Failed to set VP9E_SET_TILE_ROWS: %s", gst_vpx_error_name (status)); } +#ifdef VPX_CTRL_VP9E_SET_ROW_MT status = vpx_codec_control (&encoder->encoder, VP9E_SET_ROW_MT, vp9enc->row_mt ? 1 : 0); @@ -501,6 +508,7 @@ gst_vp9_enc_configure_encoder (GstVPXEnc * encoder, GstVideoCodecState * state) GST_DEBUG_OBJECT (encoder, "Failed to set VP9E_SET_ROW_MT: %s", gst_vpx_error_name (status)); } +#endif status = vpx_codec_control (&encoder->encoder, VP9E_SET_AQ_MODE, vp9enc->aq_mode); if (status != VPX_CODEC_OK) { @@ -531,6 +539,12 @@ gst_vp9_enc_enable_scaling (GstVPXEnc * enc) return FALSE; } +static gboolean +gst_vp9_enc_enable_tiles (GstVPXEnc * enc) +{ + return TRUE; +} + static void gst_vp9_enc_set_image_format (GstVPXEnc * enc, vpx_image_t * image) { diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.h b/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.h index 03d31d1bd23..45540d050e8 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.h +++ b/subprojects/gst-plugins-good/ext/vpx/gstvp9enc.h @@ -47,9 +47,7 @@ struct _GstVP9Enc guint tile_columns; guint tile_rows; -#ifdef VPX_CTRL_VP9E_SET_ROW_MT gboolean row_mt; -#endif GstVPXAQ aq_mode; gboolean frame_parallel_decoding; }; diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c b/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c index d5351b839a4..0b6f701e699 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.c @@ -40,6 +40,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_vpxdec_debug); #define DEFAULT_DEBLOCKING_LEVEL 4 #define DEFAULT_NOISE_LEVEL 0 #define DEFAULT_THREADS 0 +#define DEFAULT_DIRECT_RENDERING TRUE #define DEFAULT_VIDEO_CODEC_TAG NULL #define DEFAULT_CODEC_ALGO NULL @@ -50,7 +51,8 @@ enum PROP_POST_PROCESSING_FLAGS, PROP_DEBLOCKING_LEVEL, PROP_NOISE_LEVEL, - PROP_THREADS + PROP_THREADS, + PROP_DIRECT_RENDERING }; #define C_FLAGS(v) ((guint) v) @@ -94,6 +96,10 @@ gst_vpx_dec_post_processing_flags_get_type (void) #undef C_FLAGS +/* cached quark to avoid contention on the global quark table lock */ +#define META_TAG_VIDEO meta_tag_video_quark +static GQuark meta_tag_video_quark; + static void gst_vpx_dec_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_vpx_dec_get_property (GObject * object, guint prop_id, @@ -109,6 +115,8 @@ gst_vpx_dec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame); static gboolean gst_vpx_dec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query); +static gboolean gst_vpx_dec_transform_meta (GstVideoDecoder * decoder, + GstVideoCodecFrame * frame, GstMeta * meta); static void gst_vpx_dec_image_to_buffer (GstVPXDec * dec, const vpx_image_t * img, GstBuffer * buffer); @@ -167,6 +175,11 @@ gst_vpx_dec_class_init (GstVPXDecClass * klass) "Maximum number of decoding threads", 0, 16, DEFAULT_THREADS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_DIRECT_RENDERING, + g_param_spec_boolean ("direct-rendering", "Direct Rendering", + "Enable direct rendering", DEFAULT_DIRECT_RENDERING, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + base_video_decoder_class->start = GST_DEBUG_FUNCPTR (gst_vpx_dec_start); base_video_decoder_class->stop = GST_DEBUG_FUNCPTR (gst_vpx_dec_stop); base_video_decoder_class->flush = GST_DEBUG_FUNCPTR (gst_vpx_dec_flush); @@ -176,6 +189,8 @@ gst_vpx_dec_class_init (GstVPXDecClass * klass) GST_DEBUG_FUNCPTR (gst_vpx_dec_handle_frame);; base_video_decoder_class->decide_allocation = GST_DEBUG_FUNCPTR (gst_vpx_dec_decide_allocation); + base_video_decoder_class->transform_meta = + GST_DEBUG_FUNCPTR (gst_vpx_dec_transform_meta); klass->video_codec_tag = DEFAULT_VIDEO_CODEC_TAG; klass->codec_algo = DEFAULT_CODEC_ALGO; @@ -189,6 +204,8 @@ gst_vpx_dec_class_init (GstVPXDecClass * klass) GST_DEBUG_CATEGORY_INIT (gst_vpxdec_debug, "vpxdec", 0, "VPX Decoder"); + meta_tag_video_quark = g_quark_from_static_string (GST_META_TAG_VIDEO_STR); + gst_type_mark_as_plugin_api (GST_VPX_DEC_TYPE_POST_PROCESSING_FLAGS, 0); gst_type_mark_as_plugin_api (GST_TYPE_VPX_DEC, 0); } @@ -205,6 +222,7 @@ gst_vpx_dec_init (GstVPXDec * gst_vpx_dec) gst_vpx_dec->post_processing_flags = DEFAULT_POST_PROCESSING_FLAGS; gst_vpx_dec->deblocking_level = DEFAULT_DEBLOCKING_LEVEL; gst_vpx_dec->noise_level = DEFAULT_NOISE_LEVEL; + gst_vpx_dec->direct_rendering = DEFAULT_DIRECT_RENDERING; if (vpxclass->get_needs_sync_point) { gst_video_decoder_set_needs_sync_point (GST_VIDEO_DECODER (gst_vpx_dec), @@ -242,6 +260,9 @@ gst_vpx_dec_set_property (GObject * object, guint prop_id, case PROP_THREADS: dec->threads = g_value_get_uint (value); break; + case PROP_DIRECT_RENDERING: + dec->direct_rendering = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -273,6 +294,9 @@ gst_vpx_dec_get_property (GObject * object, guint prop_id, GValue * value, case PROP_THREADS: g_value_set_uint (value, dec->threads); break; + case PROP_DIRECT_RENDERING: + g_value_set_boolean (value, dec->direct_rendering); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -401,6 +425,8 @@ gst_vpx_dec_prepare_image (GstVPXDec * dec, const vpx_image_t * img) GstVideoInfo *info = &dec->output_state->info; buffer = gst_buffer_ref (frame->buffer); + gst_buffer_unmap (buffer, &frame->info); + gst_buffer_map (buffer, &frame->info, GST_MAP_READ); /* FIXME: an atomic remap would be preferable, for now we simply * remap the buffer from RW to RO when using a sysmem allocator, @@ -658,8 +684,10 @@ gst_vpx_dec_open_codec (GstVPXDec * dec, GstVideoCodecFrame * frame) gst_vpx_error_name (status)); } } - vpx_codec_set_frame_buffer_functions (&dec->decoder, - gst_vpx_dec_get_buffer_cb, gst_vpx_dec_release_buffer_cb, dec); + if (dec->direct_rendering) { + vpx_codec_set_frame_buffer_functions (&dec->decoder, + gst_vpx_dec_get_buffer_cb, gst_vpx_dec_release_buffer_cb, dec); + } dec->decoder_inited = TRUE; @@ -753,7 +781,7 @@ gst_vpx_dec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame) gst_video_decoder_drop_frame (decoder, frame); } else { gst_vpx_dec_handle_resolution_change (dec, img, fmt); - if (img->fb_priv && dec->have_video_meta) { + if (dec->direct_rendering && img->fb_priv && dec->have_video_meta) { frame->output_buffer = gst_vpx_dec_prepare_image (dec, img); ret = gst_video_decoder_finish_frame (decoder, frame); } else { @@ -809,6 +837,25 @@ gst_vpx_dec_decide_allocation (GstVideoDecoder * bdec, GstQuery * query) return TRUE; } +static gboolean +gst_vpx_dec_transform_meta (GstVideoDecoder * + decoder, GstVideoCodecFrame * frame, GstMeta * meta) +{ + const GstMetaInfo *roi_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + const GstMetaInfo *info = meta->info; + const gchar *const *tags; + + tags = gst_meta_api_type_get_tags (info->api); + + if (!tags + || info->api == roi_info->api + || (g_strv_length ((gchar **) tags) == 1 + && gst_meta_api_type_has_tag (info->api, META_TAG_VIDEO))) + return TRUE; + + return FALSE; +} + static void gst_vpx_dec_set_stream_info (GstVPXDec * dec, vpx_codec_stream_info_t * stream_info) diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.h b/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.h index e48f4e71432..3c73314aa6b 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.h +++ b/subprojects/gst-plugins-good/ext/vpx/gstvpxdec.h @@ -75,6 +75,7 @@ struct _GstVPXDec gint deblocking_level; gint noise_level; gint threads; + gboolean direct_rendering; GstVideoCodecState *input_state; GstVideoCodecState *output_state; diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c index 2676d898098..74bbb989a37 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c +++ b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.c @@ -82,6 +82,8 @@ GST_DEBUG_CATEGORY_STATIC (gst_vpxenc_debug); #define DEFAULT_LAG_IN_FRAMES 0 #define DEFAULT_THREADS 0 +#define DEFAULT_TILE_COLUMNS 6 +#define DEFAULT_TILE_ROWS 0 #define DEFAULT_H_SCALING_MODE VP8E_NORMAL #define DEFAULT_V_SCALING_MODE VP8E_NORMAL @@ -139,6 +141,8 @@ enum PROP_ERROR_RESILIENT, PROP_LAG_IN_FRAMES, PROP_THREADS, + PROP_TILE_COLUMNS, + PROP_TILE_ROWS, PROP_DEADLINE, PROP_H_SCALING_MODE, PROP_V_SCALING_MODE, @@ -598,13 +602,13 @@ gst_vpx_enc_class_init (GstVPXEncClass * klass) * Since: 1.20 */ g_object_class_install_property (gobject_class, PROP_TS_LAYER_FLAGS, - gst_param_spec_array ("temporal-scalability-layer-flags", + g_param_spec_value_array ("temporal-scalability-layer-flags", "Coding layer flags", "Sequence defining coding layer flags", - g_param_spec_flags ("flags", "Flags", "Flags", - GST_VPX_ENC_TS_LAYER_FLAGS_TYPE, 0, + g_param_spec_int ("flags", "Flags", "Flags", 0, G_MAXINT, 0, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS), G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + /** * GstVPXEnc:temporal-scalability-layer-sync-flags: * @@ -613,7 +617,7 @@ gst_vpx_enc_class_init (GstVPXEncClass * klass) * Since: 1.20 */ g_object_class_install_property (gobject_class, PROP_TS_LAYER_SYNC_FLAGS, - gst_param_spec_array ("temporal-scalability-layer-sync-flags", + g_param_spec_value_array ("temporal-scalability-layer-sync-flags", "Coding layer sync flags", "Sequence defining coding layer sync flags", g_param_spec_boolean ("flags", "Flags", "Flags", FALSE, @@ -641,6 +645,20 @@ gst_vpx_enc_class_init (GstVPXEncClass * klass) (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_DOC_SHOW_DEFAULT))); +#ifdef HAVE_VPX_1_3 + g_object_class_install_property (gobject_class, PROP_TILE_COLUMNS, + g_param_spec_int ("tile-columns", "Tile Columns", + "Number of tile columns, log2", + 0, 6, DEFAULT_TILE_COLUMNS, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_TILE_ROWS, + g_param_spec_int ("tile-rows", "Tile Rows", + "Number of tile rows, log2", + 0, 2, DEFAULT_TILE_ROWS, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); +#endif + g_object_class_install_property (gobject_class, PROP_DEADLINE, g_param_spec_int64 ("deadline", "Deadline", "Deadline per frame (usec, 0=best, 1=realtime)", @@ -781,7 +799,7 @@ gst_vpx_enc_init (GstVPXEnc * gst_vpx_enc) gst_vpx_enc->cfg.rc_end_usage = DEFAULT_RC_END_USAGE; gst_vpx_enc->cfg.rc_target_bitrate = DEFAULT_RC_TARGET_BITRATE / 1000; - gst_vpx_enc->rc_target_bitrate_auto = DEFAULT_RC_TARGET_BITRATE == 0; + gst_vpx_enc->rc_target_bitrate_auto = FALSE; gst_vpx_enc->cfg.rc_min_quantizer = DEFAULT_RC_MIN_QUANTIZER; gst_vpx_enc->cfg.rc_max_quantizer = DEFAULT_RC_MAX_QUANTIZER; gst_vpx_enc->cfg.rc_dropframe_thresh = DEFAULT_RC_DROPFRAME_THRESH; @@ -816,6 +834,8 @@ gst_vpx_enc_init (GstVPXEnc * gst_vpx_enc) gst_vpx_enc->cfg.g_error_resilient = DEFAULT_ERROR_RESILIENT; gst_vpx_enc->cfg.g_lag_in_frames = DEFAULT_LAG_IN_FRAMES; gst_vpx_enc->cfg.g_threads = DEFAULT_THREADS; + gst_vpx_enc->tile_columns = DEFAULT_TILE_COLUMNS; + gst_vpx_enc->tile_rows = DEFAULT_TILE_ROWS; gst_vpx_enc->deadline = DEFAULT_DEADLINE; gst_vpx_enc->h_scaling_mode = DEFAULT_H_SCALING_MODE; gst_vpx_enc->v_scaling_mode = DEFAULT_V_SCALING_MODE; @@ -839,6 +859,9 @@ gst_vpx_enc_init (GstVPXEnc * gst_vpx_enc) gst_vpx_enc->cfg.g_profile = DEFAULT_PROFILE; + gst_video_encoder_set_min_force_key_unit_interval (GST_VIDEO_ENCODER + (gst_vpx_enc), 1500 * GST_MSECOND); + g_mutex_init (&gst_vpx_enc->encoder_lock); } @@ -869,39 +892,41 @@ gst_vpx_enc_finalize (GObject * object) } static void -gst_vpx_enc_set_auto_bitrate (GstVPXEnc * encoder) +gst_vpx_enc_set_auto_bitrate (GstVPXEnc * encoder, + GstVideoCodecState * input_state) { - if (encoder->input_state != NULL) { - guint size; - guint pixels_per_sec; - guint target_bitrate; - guint fps_n, fps_d; - - if (GST_VIDEO_INFO_FPS_N (&encoder->input_state->info) != 0) { - fps_n = GST_VIDEO_INFO_FPS_N (&encoder->input_state->info); - fps_d = GST_VIDEO_INFO_FPS_D (&encoder->input_state->info); - } else { - /* otherwise assume 30 frames per second as a fallback */ - fps_n = 30; - fps_d = 1; - } + guint size; + guint pixels_per_sec; + guint target_bitrate; + guint fps_n, fps_d; - size = - GST_VIDEO_INFO_WIDTH (&encoder->input_state->info) * - GST_VIDEO_INFO_HEIGHT (&encoder->input_state->info); - pixels_per_sec = size * fps_n / fps_d; - target_bitrate = pixels_per_sec * encoder->bits_per_pixel; - - GST_DEBUG_OBJECT (encoder, - "Setting autobitrate for %ux%ux @ %u/%ufps %.4f = %ubps", - GST_VIDEO_INFO_WIDTH (&encoder->input_state->info), - GST_VIDEO_INFO_HEIGHT (&encoder->input_state->info), - GST_VIDEO_INFO_FPS_N (&encoder->input_state->info), - GST_VIDEO_INFO_FPS_D (&encoder->input_state->info), - encoder->bits_per_pixel, target_bitrate); - - encoder->cfg.rc_target_bitrate = target_bitrate / 1000; + if (input_state == NULL) + return; + + if (GST_VIDEO_INFO_FPS_D (&input_state->info) != 0) { + fps_n = GST_VIDEO_INFO_FPS_N (&input_state->info); + fps_d = GST_VIDEO_INFO_FPS_D (&input_state->info); + } else { + /* otherwise assume 30 frames per second as a fallback */ + fps_n = 30; + fps_d = 1; } + + size = + GST_VIDEO_INFO_WIDTH (&input_state->info) * + GST_VIDEO_INFO_HEIGHT (&input_state->info); + pixels_per_sec = size * fps_n / fps_d; + target_bitrate = pixels_per_sec * encoder->bits_per_pixel; + + GST_DEBUG_OBJECT (encoder, + "Setting autobitrate for %ux%ux @ %u/%ufps %.4f = %ubps", + GST_VIDEO_INFO_WIDTH (&input_state->info), + GST_VIDEO_INFO_HEIGHT (&input_state->info), + GST_VIDEO_INFO_FPS_N (&input_state->info), + GST_VIDEO_INFO_FPS_D (&input_state->info), + encoder->bits_per_pixel, target_bitrate); + + encoder->cfg.rc_target_bitrate = target_bitrate / 1000; } static void @@ -911,12 +936,15 @@ gst_vpx_enc_set_property (GObject * object, guint prop_id, GstVPXEnc *gst_vpx_enc; gboolean global = FALSE; vpx_codec_err_t status; + GstVPXEncClass *gst_vpx_enc_class; g_return_if_fail (GST_IS_VPX_ENC (object)); gst_vpx_enc = GST_VPX_ENC (object); + gst_vpx_enc_class = GST_VPX_ENC_GET_CLASS (gst_vpx_enc); GST_DEBUG_OBJECT (object, "gst_vpx_enc_set_property"); g_mutex_lock (&gst_vpx_enc->encoder_lock); + switch (prop_id) { case PROP_RC_END_USAGE: gst_vpx_enc->cfg.rc_end_usage = g_value_get_enum (value); @@ -924,7 +952,7 @@ gst_vpx_enc_set_property (GObject * object, guint prop_id, break; case PROP_RC_TARGET_BITRATE: if (g_value_get_int (value) == 0) { - gst_vpx_enc_set_auto_bitrate (gst_vpx_enc); + gst_vpx_enc_set_auto_bitrate (gst_vpx_enc, gst_vpx_enc->input_state); gst_vpx_enc->rc_target_bitrate_auto = TRUE; } else { gst_vpx_enc->cfg.rc_target_bitrate = g_value_get_int (value) / 1000; @@ -1077,41 +1105,36 @@ gst_vpx_enc_set_property (GObject * object, guint prop_id, break; } case PROP_TS_LAYER_FLAGS:{ - gint l = gst_value_array_get_size (value); + GValueArray *va = g_value_get_boxed (value); g_free (gst_vpx_enc->ts_layer_flags); gst_vpx_enc->n_ts_layer_flags = 0; - if (l > 0) { + if (va && va->n_values > 0) { gint i; - gst_vpx_enc->ts_layer_flags = g_new (gint, l); - - for (i = 0; i < l; i++) + gst_vpx_enc->ts_layer_flags = g_new (gint, va->n_values); + for (i = 0; i < va->n_values; i++) gst_vpx_enc->ts_layer_flags[i] = - g_value_get_flags (gst_value_array_get_value (value, i)); - gst_vpx_enc->n_ts_layer_flags = l; - } else { - gst_vpx_enc->ts_layer_flags = NULL; + g_value_get_int (g_value_array_get_nth (va, i)); + gst_vpx_enc->n_ts_layer_flags = va->n_values; } break; } case PROP_TS_LAYER_SYNC_FLAGS:{ - gint l = gst_value_array_get_size (value); + GValueArray *va = g_value_get_boxed (value); g_free (gst_vpx_enc->ts_layer_sync_flags); gst_vpx_enc->n_ts_layer_sync_flags = 0; - if (l > 0) { + if (va && va->n_values > 0) { gint i; - gst_vpx_enc->ts_layer_sync_flags = g_new (gboolean, l); - for (i = 0; i < l; i++) + gst_vpx_enc->ts_layer_sync_flags = g_new (gboolean, va->n_values); + for (i = 0; i < va->n_values; i++) gst_vpx_enc->ts_layer_sync_flags[i] = - g_value_get_boolean (gst_value_array_get_value (value, i)); - gst_vpx_enc->n_ts_layer_sync_flags = l; - } else { - gst_vpx_enc->ts_layer_sync_flags = NULL; + g_value_get_boolean (g_value_array_get_nth (va, i)); + gst_vpx_enc->n_ts_layer_sync_flags = va->n_values; } break; } @@ -1127,6 +1150,34 @@ gst_vpx_enc_set_property (GObject * object, guint prop_id, gst_vpx_enc->cfg.g_threads = g_value_get_int (value); global = TRUE; break; +#ifdef HAVE_VPX_1_3 + case PROP_TILE_COLUMNS: + gst_vpx_enc->tile_columns = g_value_get_int (value); + if (gst_vpx_enc_class->enable_tiles (gst_vpx_enc) && gst_vpx_enc->inited) { + status = + vpx_codec_control (&gst_vpx_enc->encoder, VP9E_SET_TILE_COLUMNS, + gst_vpx_enc->tile_columns); + if (status != VPX_CODEC_OK) { + GST_WARNING_OBJECT (gst_vpx_enc, + "Failed to set VP9E_SET_TILE_COLUMNS: %s", + gst_vpx_error_name (status)); + } + } + break; + case PROP_TILE_ROWS: + gst_vpx_enc->tile_rows = g_value_get_int (value); + if (gst_vpx_enc_class->enable_tiles (gst_vpx_enc) && gst_vpx_enc->inited) { + status = + vpx_codec_control (&gst_vpx_enc->encoder, VP9E_SET_TILE_ROWS, + gst_vpx_enc->tile_rows); + if (status != VPX_CODEC_OK) { + GST_WARNING_OBJECT (gst_vpx_enc, + "Failed to set VP9E_SET_TILE_ROWS: %s", + gst_vpx_error_name (status)); + } + } + break; +#endif case PROP_DEADLINE: gst_vpx_enc->deadline = g_value_get_int64 (value); break; @@ -1314,7 +1365,7 @@ gst_vpx_enc_set_property (GObject * object, guint prop_id, case PROP_BITS_PER_PIXEL: gst_vpx_enc->bits_per_pixel = g_value_get_float (value); if (gst_vpx_enc->rc_target_bitrate_auto) { - gst_vpx_enc_set_auto_bitrate (gst_vpx_enc); + gst_vpx_enc_set_auto_bitrate (gst_vpx_enc, gst_vpx_enc->input_state); global = TRUE; } break; @@ -1482,28 +1533,46 @@ gst_vpx_enc_get_property (GObject * object, guint prop_id, GValue * value, break; } case PROP_TS_LAYER_FLAGS:{ - gint i; + GValueArray *va; - for (i = 0; i < gst_vpx_enc->n_ts_layer_flags; i++) { - GValue v = { 0, }; + if (gst_vpx_enc->n_ts_layer_flags == 0) { + g_value_set_boxed (value, NULL); + } else { + gint i; - g_value_init (&v, GST_VPX_ENC_TS_LAYER_FLAGS_TYPE); - g_value_set_flags (&v, gst_vpx_enc->ts_layer_flags[i]); - gst_value_array_append_value (value, &v); - g_value_unset (&v); + va = g_value_array_new (gst_vpx_enc->n_ts_layer_flags); + for (i = 0; i < gst_vpx_enc->n_ts_layer_flags; i++) { + GValue v = { 0, }; + + g_value_init (&v, G_TYPE_INT); + g_value_set_int (&v, gst_vpx_enc->ts_layer_flags[i]); + g_value_array_append (va, &v); + g_value_unset (&v); + } + g_value_set_boxed (value, va); + g_value_array_free (va); } break; } case PROP_TS_LAYER_SYNC_FLAGS:{ - gint i; + GValueArray *va; + + if (gst_vpx_enc->n_ts_layer_sync_flags == 0) { + g_value_set_boxed (value, NULL); + } else { + gint i; - for (i = 0; i < gst_vpx_enc->n_ts_layer_sync_flags; i++) { - GValue v = { 0, }; + va = g_value_array_new (gst_vpx_enc->n_ts_layer_sync_flags); + for (i = 0; i < gst_vpx_enc->n_ts_layer_sync_flags; i++) { + GValue v = { 0, }; - g_value_init (&v, G_TYPE_BOOLEAN); - g_value_set_boolean (&v, gst_vpx_enc->ts_layer_sync_flags[i]); - gst_value_array_append_value (value, &v); - g_value_unset (&v); + g_value_init (&v, G_TYPE_BOOLEAN); + g_value_set_boolean (&v, gst_vpx_enc->ts_layer_sync_flags[i]); + g_value_array_append (va, &v); + g_value_unset (&v); + } + g_value_set_boxed (value, va); + g_value_array_free (va); } break; } @@ -1516,6 +1585,14 @@ gst_vpx_enc_get_property (GObject * object, guint prop_id, GValue * value, case PROP_THREADS: g_value_set_int (value, gst_vpx_enc->cfg.g_threads); break; +#ifdef HAVE_VPX_1_3 + case PROP_TILE_COLUMNS: + g_value_set_int (value, gst_vpx_enc->tile_columns); + break; + case PROP_TILE_ROWS: + g_value_set_int (value, gst_vpx_enc->tile_rows); + break; +#endif case PROP_DEADLINE: g_value_set_int64 (value, gst_vpx_enc->deadline); break; @@ -1611,10 +1688,6 @@ gst_vpx_enc_destroy_encoder (GstVPXEnc * encoder) encoder->cfg.rc_twopass_stats_in.buf = NULL; encoder->cfg.rc_twopass_stats_in.sz = 0; } - - encoder->last_pts = GST_CLOCK_TIME_NONE; - encoder->last_input_duration = GST_CLOCK_TIME_NONE; - g_mutex_unlock (&encoder->encoder_lock); } @@ -1743,8 +1816,6 @@ gst_vpx_enc_set_format (GstVideoEncoder * video_encoder, vpx_codec_destroy (&encoder->encoder); encoder->inited = FALSE; encoder->multipass_cache_idx++; - encoder->last_pts = GST_CLOCK_TIME_NONE; - encoder->last_input_duration = GST_CLOCK_TIME_NONE; } else { g_mutex_lock (&encoder->encoder_lock); } @@ -1765,6 +1836,10 @@ gst_vpx_enc_set_format (GstVideoEncoder * video_encoder, encoder->cfg.g_w = GST_VIDEO_INFO_WIDTH (info); encoder->cfg.g_h = GST_VIDEO_INFO_HEIGHT (info); + /* Scale default bitrate to our size */ + if (encoder->rc_target_bitrate_auto) + gst_vpx_enc_set_auto_bitrate (encoder, state); + if (encoder->timebase_n != 0 && encoder->timebase_d != 0) { GST_DEBUG_OBJECT (video_encoder, "Using timebase configuration"); encoder->cfg.g_timebase.num = encoder->timebase_n; @@ -1847,6 +1922,25 @@ gst_vpx_enc_set_format (GstVideoEncoder * video_encoder, gst_vpx_error_name (status)); } } +#ifdef HAVE_VPX_1_3 + if (vpx_enc_class->enable_tiles (encoder)) { + status = + vpx_codec_control (&encoder->encoder, VP9E_SET_TILE_COLUMNS, + encoder->tile_columns); + if (status != VPX_CODEC_OK) { + GST_WARNING_OBJECT (encoder, "Failed to set VP9E_SET_TILE_COLUMNS: %s", + gst_vpx_error_name (status)); + } + + status = + vpx_codec_control (&encoder->encoder, VP9E_SET_TILE_ROWS, + encoder->tile_rows); + if (status != VPX_CODEC_OK) { + GST_WARNING_OBJECT (encoder, "Failed to set VP9E_SET_TILE_ROWS: %s", + gst_vpx_error_name (status)); + } + } +#endif status = vpx_codec_control (&encoder->encoder, VP8E_SET_CPUUSED, @@ -1950,10 +2044,6 @@ gst_vpx_enc_set_format (GstVideoEncoder * video_encoder, gst_video_codec_state_unref (encoder->input_state); encoder->input_state = gst_video_codec_state_ref (state); - /* Scale default bitrate to our size */ - if (encoder->rc_target_bitrate_auto) - gst_vpx_enc_set_auto_bitrate (encoder); - /* prepare cached image buffer setup */ image = &encoder->image; memset (image, 0, sizeof (*image)); @@ -2040,24 +2130,11 @@ gst_vpx_enc_process (GstVPXEnc * encoder) /* discard older frames that were dropped by libvpx */ frame = NULL; do { - GstClockTime pts_rt; - if (frame) gst_video_encoder_finish_frame (video_encoder, frame); frame = gst_video_encoder_get_oldest_frame (video_encoder); - if (!frame) { - GST_WARNING_OBJECT (encoder, - "vpx pts %" G_GINT64_FORMAT - " does not match input frames, discarding", pkt->data.frame.pts); - goto out; - } - - pts_rt = - gst_segment_to_running_time (&video_encoder->input_segment, - GST_FORMAT_TIME, frame->pts); - pts = - gst_util_uint64_scale (pts_rt, + gst_util_uint64_scale (frame->pts, encoder->cfg.g_timebase.den, encoder->cfg.g_timebase.num * (GstClockTime) GST_SECOND); GST_TRACE_OBJECT (encoder, "vpx pts: %" G_GINT64_FORMAT @@ -2122,8 +2199,6 @@ gst_vpx_enc_process (GstVPXEnc * encoder) pkt = vpx_codec_get_cx_data (&encoder->encoder, &iter); } - -out: g_mutex_unlock (&encoder->encoder_lock); return ret; @@ -2138,20 +2213,14 @@ gst_vpx_enc_drain (GstVideoEncoder * video_encoder) vpx_codec_err_t status; gint64 deadline; vpx_codec_pts_t pts; - GstClockTime gst_pts = 0; encoder = GST_VPX_ENC (video_encoder); g_mutex_lock (&encoder->encoder_lock); deadline = encoder->deadline; - if (GST_CLOCK_TIME_IS_VALID (encoder->last_pts)) - gst_pts = encoder->last_pts; - if (GST_CLOCK_TIME_IS_VALID (encoder->last_input_duration)) - gst_pts += encoder->last_input_duration; - pts = - gst_util_uint64_scale (gst_pts, + gst_util_uint64_scale (encoder->last_pts, encoder->cfg.g_timebase.den, encoder->cfg.g_timebase.num * (GstClockTime) GST_SECOND); @@ -2249,7 +2318,6 @@ gst_vpx_enc_handle_frame (GstVideoEncoder * video_encoder, int flags = 0; vpx_image_t *image; GstVideoFrame vframe; - GstClockTime pts_rt; vpx_codec_pts_t pts; unsigned long duration; GstVPXEncClass *vpx_enc_class; @@ -2274,28 +2342,11 @@ gst_vpx_enc_handle_frame (GstVideoEncoder * video_encoder, } g_mutex_lock (&encoder->encoder_lock); - - /* the input pts needs to be strictly increasing, see vpx_codec_encode() doc, so convert it to - * running time as we don't want to reset the encoder for each segment. */ - pts_rt = - gst_segment_to_running_time (&video_encoder->input_segment, - GST_FORMAT_TIME, frame->pts); - - /* vpx_codec_encode() enforces us to pass strictly increasing pts */ - if (GST_CLOCK_TIME_IS_VALID (encoder->last_pts) - && pts_rt <= encoder->last_pts) { - GST_WARNING_OBJECT (encoder, - "decreasing pts %" GST_TIME_FORMAT " previous buffer was %" - GST_TIME_FORMAT " enforce increasing pts", GST_TIME_ARGS (pts_rt), - GST_TIME_ARGS (encoder->last_pts)); - pts_rt = encoder->last_pts + 1; - } - pts = - gst_util_uint64_scale (pts_rt, + gst_util_uint64_scale (frame->pts, encoder->cfg.g_timebase.den, encoder->cfg.g_timebase.num * (GstClockTime) GST_SECOND); - encoder->last_pts = pts_rt; + encoder->last_pts = frame->pts; if (frame->duration != GST_CLOCK_TIME_NONE) { duration = @@ -2303,7 +2354,7 @@ gst_vpx_enc_handle_frame (GstVideoEncoder * video_encoder, encoder->cfg.g_timebase.num * (GstClockTime) GST_SECOND); if (duration > 0) { - encoder->last_input_duration = frame->duration; + encoder->last_pts += frame->duration; } else { /* We force the path ignoring the duration if we end up with a zero * value for duration after scaling (e.g. duration value too small) */ diff --git a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.h b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.h index d9c40e5440e..188b1b42f46 100644 --- a/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.h +++ b/subprojects/gst-plugins-good/ext/vpx/gstvpxenc.h @@ -87,6 +87,8 @@ struct _GstVPXEnc gint64 deadline; /* Controls */ + unsigned int tile_columns; + unsigned int tile_rows; VPX_SCALING_MODE h_scaling_mode; VPX_SCALING_MODE v_scaling_mode; int cpu_used; @@ -132,6 +134,8 @@ struct _GstVPXEncClass gboolean (*enable_scaling) (GstVPXEnc *enc); /*called from set_format with lock taken*/ gboolean (*configure_encoder) (GstVPXEnc *enc, GstVideoCodecState *state); + /*enabled tiles*/ + gboolean (*enable_tiles) (GstVPXEnc *enc); /*set image format info*/ void (*set_image_format) (GstVPXEnc *enc, vpx_image_t *image); /*get new simple caps*/ diff --git a/subprojects/gst-plugins-good/ext/vpx/meson.build b/subprojects/gst-plugins-good/ext/vpx/meson.build index 9b8b34c67e3..8acff35530e 100644 --- a/subprojects/gst-plugins-good/ext/vpx/meson.build +++ b/subprojects/gst-plugins-good/ext/vpx/meson.build @@ -18,10 +18,10 @@ vpx_features = [ ] vpx_option = get_option('vpx') -vpx_dep = dependency('vpx', version : '>=1.7.0', required : vpx_option, allow_fallback: true) +vpx_dep = dependency('vpx-avx', version : '>=1.5.0', required : vpx_option) if vpx_dep.found() - vpx_args = [] + vpx_args = ['-DHAVE_VPX_1_3'] foreach f : vpx_features header = f.get(0) codec_iface = f.get(1) @@ -93,3 +93,46 @@ if vpx_dep.found() env.prepend('GST_PRESET_PATH', meson.current_source_dir()) meson.add_devenv(env) endif + + +if get_option('default_library') == 'static' + subdir_done() +endif + +#Pexip spesific + +pex_isas = [ + ['avx', ['-DGST_ISA_AVX'] ], + ['avx2', ['-DGST_ISA_AVX2'] ], + ['avx512', ['-DGST_ISA_AVX512'] ], +] + +vpx_args = [ + '-DHAVE_VPX_1_3', + '-DHAVE_VPX_1_4', + '-DHAVE_VP8_ENCODER', + '-DHAVE_VP8_DECODER', + '-DHAVE_VP9_ENCODER', + '-DHAVE_VP9_DECODER', +] + +plugins_install_dir_isa = '@0@-'.format(plugins_install_dir) + +foreach isa_flags : pex_isas + isa = isa_flags[0] + flags = isa_flags[1] + + vpxisa_dep = dependency('vpx-@0@'.format(isa), required : false) + + if vpxisa_dep.found() + gstvpxisa = library('gstvpx_' + isa, + vpx_sources, gstvpx_enums, + c_args : gst_plugins_good_args + vpx_args + flags, + include_directories: [configinc], + dependencies : [gstbase_dep, gsttag_dep, gstvideo_dep, vpxisa_dep], + install : true, + install_dir : plugins_install_dir_isa + isa, + ) + plugins += [gstvpxisa] + endif +endforeach diff --git a/subprojects/gst-plugins-good/ext/vpx/plugin.c b/subprojects/gst-plugins-good/ext/vpx/plugin.c index 1887aee70e2..e1472c550e2 100644 --- a/subprojects/gst-plugins-good/ext/vpx/plugin.c +++ b/subprojects/gst-plugins-good/ext/vpx/plugin.c @@ -33,26 +33,41 @@ plugin_init (GstPlugin * plugin) gboolean ret = FALSE; #ifdef HAVE_VP8_DECODER - ret |= GST_ELEMENT_REGISTER (vp8dec, plugin); + if (!g_type_from_name ("GstVP8Dec")) + ret |= GST_ELEMENT_REGISTER (vp8dec, plugin); #endif #ifdef HAVE_VP8_ENCODER - ret |= GST_ELEMENT_REGISTER (vp8enc, plugin); + if (!g_type_from_name ("GstVP8Enc")) + ret |= GST_ELEMENT_REGISTER (vp8enc, plugin); #endif #ifdef HAVE_VP9_DECODER - ret |= GST_ELEMENT_REGISTER (vp9dec, plugin); + if (!g_type_from_name ("GstVP9Dec")) + ret |= GST_ELEMENT_REGISTER (vp9dec, plugin); #endif #ifdef HAVE_VP9_ENCODER - ret |= GST_ELEMENT_REGISTER (vp9enc, plugin); + if (!g_type_from_name ("GstVP9Enc")) + ret |= GST_ELEMENT_REGISTER (vp9enc, plugin); #endif return ret; } +/* Pexip Spesific */ +#if defined (GST_ISA_AVX) +# define GST_ISA_SUFFIX(name) G_PASTE (name, _avx) +#elif defined (GST_ISA_AVX2) +# define GST_ISA_SUFFIX(name) G_PASTE (name, _avx2) +#elif defined (GST_ISA_AVX512) +# define GST_ISA_SUFFIX(name) G_PASTE (name, _avx512) +#else +# define GST_ISA_SUFFIX(name) name +#endif + GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, GST_VERSION_MINOR, - vpx, + GST_ISA_SUFFIX (vpx), "VP8 plugin", plugin_init, VERSION, "LGPL", GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/subprojects/gst-plugins-good/gst/flv/gstflvdemux.c b/subprojects/gst-plugins-good/gst/flv/gstflvdemux.c index c4df091f079..665b538cfa5 100644 --- a/subprojects/gst-plugins-good/gst/flv/gstflvdemux.c +++ b/subprojects/gst-plugins-good/gst/flv/gstflvdemux.c @@ -89,6 +89,12 @@ static GstStaticPadTemplate video_src_template = "video/x-h264, stream-format=avc;") ); +enum +{ + PROP_00, /* PROP_0 already used by included gstindex.c */ + PROP_NO_MORE_PADS_THRESHOLD +}; + #define gst_flv_demux_parent_class parent_class G_DEFINE_TYPE (GstFlvDemux, gst_flv_demux, GST_TYPE_ELEMENT); GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (flvdemux, "flvdemux", @@ -103,7 +109,7 @@ GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (flvdemux, "flvdemux", #define RESYNC_THRESHOLD 2000 /* how much stream time to wait for audio tags to appear after we have video, or vice versa */ -#define NO_MORE_PADS_THRESHOLD (6 * GST_SECOND) +#define DEFAULT_NO_MORE_PADS_THRESHOLD -1 static gboolean flv_demux_handle_seek_push (GstFlvDemux * demux, GstEvent * event); @@ -1396,9 +1402,10 @@ gst_flv_demux_parse_tag_audio (GstFlvDemux * demux, GstBuffer * buffer) demux->audio_first_ts = GST_BUFFER_TIMESTAMP (outbuf); } - if (G_UNLIKELY (!demux->streams_aware && !demux->no_more_pads - && (GST_CLOCK_DIFF (demux->audio_start, - GST_BUFFER_TIMESTAMP (outbuf)) > NO_MORE_PADS_THRESHOLD))) { + if (G_UNLIKELY (!demux->streams_aware && !demux->no_more_pads && + (demux->no_more_pads_threshold != -1) && + (GST_CLOCK_DIFF (demux->audio_start, + GST_BUFFER_TIMESTAMP (outbuf)) > demux->no_more_pads_threshold))) { GST_DEBUG_OBJECT (demux, "Signalling no-more-pads because no video stream was found" " after 6 seconds of audio"); @@ -1874,9 +1881,10 @@ gst_flv_demux_parse_tag_video (GstFlvDemux * demux, GstBuffer * buffer) demux->video_first_ts = GST_BUFFER_TIMESTAMP (outbuf); } - if (G_UNLIKELY (!demux->streams_aware && !demux->no_more_pads - && (GST_CLOCK_DIFF (demux->video_start, - GST_BUFFER_TIMESTAMP (outbuf)) > NO_MORE_PADS_THRESHOLD))) { + if (G_UNLIKELY (!demux->streams_aware && !demux->no_more_pads && + (demux->no_more_pads_threshold != -1) && + (GST_CLOCK_DIFF (demux->video_start, + GST_BUFFER_TIMESTAMP (outbuf)) > demux->no_more_pads_threshold))) { GST_DEBUG_OBJECT (demux, "Signalling no-more-pads because no audio stream was found" " after 6 seconds of video"); @@ -3800,6 +3808,38 @@ gst_flv_demux_get_index (GstElement * element) return result; } +static void +gst_flv_demux_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstFlvDemux *demux = GST_FLV_DEMUX (object); + + switch (prop_id) { + case PROP_NO_MORE_PADS_THRESHOLD: + g_value_set_int64 (value, demux->no_more_pads_threshold); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_flv_demux_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstFlvDemux *demux = GST_FLV_DEMUX (object); + + switch (prop_id) { + case PROP_NO_MORE_PADS_THRESHOLD: + demux->no_more_pads_threshold = g_value_get_int64 (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + static void gst_flv_demux_dispose (GObject * object) { @@ -3882,6 +3922,8 @@ gst_flv_demux_class_init (GstFlvDemuxClass * klass) GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass); GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + gobject_class->get_property = gst_flv_demux_get_property; + gobject_class->set_property = gst_flv_demux_set_property; gobject_class->dispose = gst_flv_demux_dispose; gstelement_class->change_state = @@ -3901,6 +3943,13 @@ gst_flv_demux_class_init (GstFlvDemuxClass * klass) gst_element_class_set_static_metadata (gstelement_class, "FLV Demuxer", "Codec/Demuxer", "Demux FLV feeds into digital streams", "Julien Moutte "); + + g_object_class_install_property (gobject_class, PROP_NO_MORE_PADS_THRESHOLD, + g_param_spec_int64 ("no-more-pads-threshold", "No-More-Pads Threshold", + "How much stream time to wait for audio tags to appear after we " + "have video, or vice versa (-1 waits forever)", + -1, G_MAXINT64, DEFAULT_NO_MORE_PADS_THRESHOLD, + G_PARAM_CONSTRUCT | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); } static void diff --git a/subprojects/gst-plugins-good/gst/flv/gstflvdemux.h b/subprojects/gst-plugins-good/gst/flv/gstflvdemux.h index 71584577177..d04332229f7 100644 --- a/subprojects/gst-plugins-good/gst/flv/gstflvdemux.h +++ b/subprojects/gst-plugins-good/gst/flv/gstflvdemux.h @@ -162,6 +162,9 @@ struct _GstFlvDemux gboolean audio_done; gint64 from_offset; gint64 to_offset; + + /* properties */ + gint64 no_more_pads_threshold; }; struct _GstFlvDemuxClass diff --git a/subprojects/gst-plugins-good/gst/flv/gstflvmux.c b/subprojects/gst-plugins-good/gst/flv/gstflvmux.c index 37f7d6d5a39..304c7f64de5 100644 --- a/subprojects/gst-plugins-good/gst/flv/gstflvmux.c +++ b/subprojects/gst-plugins-good/gst/flv/gstflvmux.c @@ -1284,7 +1284,7 @@ gst_flv_mux_buffer_to_tag_internal (GstFlvMux * mux, GstBuffer * buffer, * also between the audio & video streams. */ if (dts < mux->last_dts && mux->enforce_increasing_timestamps) { - GST_WARNING_OBJECT (pad, "Got backwards dts! (%" GST_TIME_FORMAT + GST_INFO_OBJECT (pad, "Got backwards dts! (%" GST_TIME_FORMAT " < %" GST_TIME_FORMAT ")", GST_TIME_ARGS (dts * GST_MSECOND), GST_TIME_ARGS (mux->last_dts * GST_MSECOND)); dts = mux->last_dts; diff --git a/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c b/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c index 3923456d086..02b55836612 100644 --- a/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c +++ b/subprojects/gst-plugins-good/gst/isomp4/qtdemux.c @@ -115,6 +115,12 @@ GST_DEBUG_CATEGORY (qtdemux_debug); typedef struct _QtDemuxCencSampleSetInfo QtDemuxCencSampleSetInfo; typedef struct _QtDemuxAavdEncryptionInfo QtDemuxAavdEncryptionInfo; +enum +{ + PROP_0, + PROP_MAX_AUDIO_SAMPLES +}; + /* Macros for converting to/from timescale */ #define QTSTREAMTIME_TO_GSTTIME(stream, value) (gst_util_uint64_scale((value), GST_SECOND, (stream)->timescale)) #define GSTTIME_TO_QTSTREAMTIME(stream, value) (gst_util_uint64_scale((value), (stream)->timescale, GST_SECOND)) @@ -402,6 +408,42 @@ static void gst_qtdemux_reset (GstQTDemux * qtdemux, gboolean hard); static void qtdemux_clear_protection_events_on_all_streams (GstQTDemux * qtdemux); +static void +gst_qtdemux_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstQTDemux *qtdemux = GST_QTDEMUX (object); + + GST_OBJECT_LOCK (qtdemux); + switch (prop_id) { + case PROP_MAX_AUDIO_SAMPLES: + qtdemux->max_audio_samples = g_value_get_uint (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + GST_OBJECT_UNLOCK (qtdemux); +} + +static void +gst_qtdemux_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstQTDemux *qtdemux = GST_QTDEMUX (object); + + GST_OBJECT_LOCK (qtdemux); + switch (prop_id) { + case PROP_MAX_AUDIO_SAMPLES: + g_value_set_uint (value, qtdemux->max_audio_samples); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + GST_OBJECT_UNLOCK (qtdemux); +} + static void gst_qtdemux_class_init (GstQTDemuxClass * klass) { @@ -416,6 +458,9 @@ gst_qtdemux_class_init (GstQTDemuxClass * klass) gobject_class->dispose = gst_qtdemux_dispose; gobject_class->finalize = gst_qtdemux_finalize; + gobject_class->get_property = gst_qtdemux_get_property; + gobject_class->set_property = gst_qtdemux_set_property; + gstelement_class->change_state = GST_DEBUG_FUNCPTR (gst_qtdemux_change_state); #if 0 gstelement_class->set_index = GST_DEBUG_FUNCPTR (gst_qtdemux_set_index); @@ -438,6 +483,11 @@ gst_qtdemux_class_init (GstQTDemuxClass * klass) "Demultiplex a QuickTime file into audio and video streams", "David Schleef , Wim Taymans "); + g_object_class_install_property (gobject_class, PROP_MAX_AUDIO_SAMPLES, + g_param_spec_uint ("max-audio-samples", "Max audio samples", + "Maximum raw audio samples per buffer", 1, G_MAXUINT, 4096, + G_PARAM_CONSTRUCT | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + GST_DEBUG_CATEGORY_INIT (qtdemux_debug, "qtdemux", 0, "qtdemux plugin"); gst_riff_init (); } @@ -11699,7 +11749,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak) GNode *stbl; GNode *stsd; GNode *mp4a; - GNode *mp4v; + GNode *mp4v = NULL; GNode *esds; GNode *tref; GNode *udta; @@ -13582,6 +13632,7 @@ qtdemux_parse_trak (GstQTDemux * qtdemux, GNode * trak) goto corrupt_file; } + gst_caps_unref (entry->caps); entry->caps = gst_codec_utils_opus_create_caps (rate, n_channels, channel_mapping_family, stream_count, coupled_count, channel_mapping); @@ -16181,7 +16232,8 @@ qtdemux_audio_caps (GstQTDemux * qtdemux, QtDemuxStream * stream, if (g_str_has_prefix (name, "audio/x-raw")) { stream->need_clip = TRUE; stream->min_buffer_size = 1024 * entry->bytes_per_frame; - stream->max_buffer_size = 4096 * entry->bytes_per_frame; + stream->max_buffer_size = + qtdemux->max_audio_samples * entry->bytes_per_frame; GST_DEBUG ("setting min/max buffer sizes to %d/%d", stream->min_buffer_size, stream->max_buffer_size); } diff --git a/subprojects/gst-plugins-good/gst/isomp4/qtdemux.h b/subprojects/gst-plugins-good/gst/isomp4/qtdemux.h index 6e7f64b91c5..c8922349807 100644 --- a/subprojects/gst-plugins-good/gst/isomp4/qtdemux.h +++ b/subprojects/gst-plugins-good/gst/isomp4/qtdemux.h @@ -309,6 +309,10 @@ struct _GstQTDemux { * fields. */ gboolean received_seek; gboolean first_moof_already_parsed; + + /** Maximum number of audio samples per buffer when demuxing raw audio. + * Used to determine max buffer size for raw audio streams. */ + guint max_audio_samples; }; struct _GstQTDemuxClass { diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpg722depay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpg722depay.c index 060c7e1c8c5..9461b40a7b7 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpg722depay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpg722depay.c @@ -53,7 +53,7 @@ static GstStaticPadTemplate gst_rtp_g722_depay_sink_template = "encoding-name = (string) \"G722\";" "application/x-rtp, " "media = (string) \"audio\", " - "payload = (int) " GST_RTP_PAYLOAD_G722_STRING ", " + "payload = (int) " GST_RTP_PAYLOAD_DYNAMIC_STRING ", " "clock-rate = (int) [ 1, MAX ]" /* "channels = (int) [1, MAX]" */ /* "emphasis = (string) ANY" */ diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph261depay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph261depay.c index 23a888a4eac..3d0af365b94 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtph261depay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph261depay.c @@ -178,8 +178,8 @@ gst_rtp_h261_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) * stream is intra coded. */ if (header->i) GST_BUFFER_FLAG_UNSET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT); - else - GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT); + //else + // GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT); GST_DEBUG_OBJECT (depay, "Pushing out a buffer of %u bytes", avail); depay->start = FALSE; diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph263ppay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph263ppay.c index b6d8040aea6..5eb13ea6bf2 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtph263ppay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph263ppay.c @@ -494,7 +494,6 @@ gst_rtp_h263p_pay_sink_getcaps (GstRTPBasePayload * payload, GstPad * pad, caps = gst_caps_merge_structure (caps, new_s); } } else { - gboolean f = FALSE, i = FALSE, j = FALSE, t = FALSE; /* FIXME: ffmpeg support the Appendix K too, how do we express it ? * guint k; */ @@ -503,52 +502,45 @@ gst_rtp_h263p_pay_sink_getcaps (GstRTPBasePayload * payload, GstPad * pad, "variant", G_TYPE_STRING, "itu", NULL); gboolean added = FALSE; + GValue list = { 0 }; + GValue vstr = { 0 }; + + g_value_init (&list, GST_TYPE_LIST); + g_value_init (&vstr, G_TYPE_STRING); + + g_value_set_static_string (&vstr, "h263"); + gst_value_list_append_value (&list, &vstr); + g_value_set_static_string (&vstr, "h263p"); + gst_value_list_append_value (&list, &vstr); + g_value_unset (&vstr); + + gst_structure_set_value (new_s, "h263version", &list); + g_value_unset (&list); str = gst_structure_get_string (s, "f"); - if (str && !strcmp (str, "1")) - f = TRUE; + if (str) { + gst_structure_set (new_s, "annex-f", + G_TYPE_BOOLEAN, !strcmp (str, "1"), NULL); + } str = gst_structure_get_string (s, "i"); - if (str && !strcmp (str, "1")) - i = TRUE; + if (str) { + gst_structure_set (new_s, "annex-i", + G_TYPE_BOOLEAN, !strcmp (str, "1"), NULL); + } str = gst_structure_get_string (s, "j"); - if (str && !strcmp (str, "1")) - j = TRUE; + if (str) { + gst_structure_set (new_s, "annex-j", + G_TYPE_BOOLEAN, !strcmp (str, "1"), NULL); + } str = gst_structure_get_string (s, "t"); - if (str && !strcmp (str, "1")) - t = TRUE; - - if (f || i || j || t) { - GValue list = { 0 }; - GValue vstr = { 0 }; - - g_value_init (&list, GST_TYPE_LIST); - g_value_init (&vstr, G_TYPE_STRING); - - g_value_set_static_string (&vstr, "h263"); - gst_value_list_append_value (&list, &vstr); - g_value_set_static_string (&vstr, "h263p"); - gst_value_list_append_value (&list, &vstr); - g_value_unset (&vstr); - - gst_structure_set_value (new_s, "h263version", &list); - g_value_unset (&list); - } else { - gst_structure_set (new_s, "h263version", G_TYPE_STRING, "h263", NULL); + if (str) { + gst_structure_set (new_s, "annex-t", + G_TYPE_BOOLEAN, !strcmp (str, "1"), NULL); } - if (!f) - gst_structure_set (new_s, "annex-f", G_TYPE_BOOLEAN, FALSE, NULL); - if (!i) - gst_structure_set (new_s, "annex-i", G_TYPE_BOOLEAN, FALSE, NULL); - if (!j) - gst_structure_set (new_s, "annex-j", G_TYPE_BOOLEAN, FALSE, NULL); - if (!t) - gst_structure_set (new_s, "annex-t", G_TYPE_BOOLEAN, FALSE, NULL); - - str = gst_structure_get_string (s, "custom"); if (str) { unsigned int xmax, ymax, mpi; diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpopuspay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpopuspay.c index e62868c3767..840457412d8 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpopuspay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpopuspay.c @@ -328,6 +328,7 @@ gst_rtp_opus_pay_handle_buffer (GstRTPBasePayload * basepayload, GstRtpOPUSPay *self = GST_RTP_OPUS_PAY_CAST (basepayload); GstBuffer *outbuf; GstClockTime pts, dts, duration; + guint64 offset; /* DTX packets are zero-length frames, with a 1 or 2-bytes header */ if (self->dtx && gst_buffer_get_size (buffer) <= 2) { @@ -341,6 +342,7 @@ gst_rtp_opus_pay_handle_buffer (GstRTPBasePayload * basepayload, pts = GST_BUFFER_PTS (buffer); dts = GST_BUFFER_DTS (buffer); duration = GST_BUFFER_DURATION (buffer); + offset = GST_BUFFER_OFFSET (buffer); outbuf = gst_rtp_base_payload_allocate_output_buffer (basepayload, 0, 0, 0); @@ -351,6 +353,7 @@ gst_rtp_opus_pay_handle_buffer (GstRTPBasePayload * basepayload, GST_BUFFER_PTS (outbuf) = pts; GST_BUFFER_DTS (outbuf) = dts; GST_BUFFER_DURATION (outbuf) = duration; + GST_BUFFER_OFFSET (outbuf) = offset; if (self->marker) { GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.c index 708bceb688c..2bf9eb46f22 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.c @@ -34,10 +34,6 @@ GST_DEBUG_CATEGORY_STATIC (gst_rtp_vp8_depay_debug); #define GST_CAT_DEFAULT gst_rtp_vp8_depay_debug static void gst_rtp_vp8_depay_dispose (GObject * object); -static void gst_rtp_vp8_depay_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec); -static void gst_rtp_vp8_depay_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec); static GstBuffer *gst_rtp_vp8_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp); static GstStateChangeReturn gst_rtp_vp8_depay_change_state (GstElement * @@ -66,29 +62,216 @@ GST_STATIC_PAD_TEMPLATE ("sink", "media = (string) \"video\"," "encoding-name = (string) { \"VP8\", \"VP8-DRAFT-IETF-01\" }")); -#define DEFAULT_WAIT_FOR_KEYFRAME FALSE -#define DEFAULT_REQUEST_KEYFRAME FALSE - enum { PROP_0, PROP_WAIT_FOR_KEYFRAME, - PROP_REQUEST_KEYFRAME, + PROP_HIDE_PICTURE_ID_GAP, }; +typedef struct _GstVP8PacketInfo +{ + guint size; + gboolean frame_start; + gboolean end_of_frame; + guint8 part_start; + guint8 is_non_ref_frame; + gboolean temporally_scaled; + guint8 layer_sync; + guint8 part_idx; + guint32 picture_id; + guint8 tl0picidx; + guint8 tid; + guint8 temporal_key_idx; + guint8 hdrsize; + GstClockTime pts; +} GstVP8PacketInfo; + #define PICTURE_ID_NONE (UINT_MAX) #define IS_PICTURE_ID_15BITS(pid) (((guint)(pid) & 0x8000) != 0) +#define DEFAULT_WAIT_FOR_KEYFRAME FALSE +#define DEFAULT_HIDE_PICTURE_ID_GAP FALSE + +// VP8 Payload Descriptor Format +// (see RFC:7741 Section-4.2) +// 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 +// +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ +// |X|R|N|S|R| PID | (REQUIRED) |X|R|N|S|R| PID | (REQUIRED) +// +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ +// X: |I|L|T|K| RSV | (OPTIONAL) X: |I|L|T|K| RSV | (OPTIONAL) +// +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ +// I: |M| PictureID | (OPTIONAL) I: |M| PictureID | (OPTIONAL) +// +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ +// L: | TL0PICIDX | (OPTIONAL) | PictureID | +// +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ +// T/K: |TID|Y| KEYIDX | (OPTIONAL) L: | TL0PICIDX | (OPTIONAL) +// +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ +// T/K: |TID|Y| KEYIDX | (OPTIONAL) +// +-+-+-+-+-+-+-+-+ +static gboolean +gst_rtp_vp8_depay_parse_header (GstRTPBuffer * rtp, GstVP8PacketInfo * out) +{ + guint8 has_ext_ctrl_bits = FALSE; + guint8 *data = gst_rtp_buffer_get_payload (rtp); + guint size = gst_rtp_buffer_get_payload_len (rtp); + GstBitReader br = GST_BIT_READER_INIT (data, size); + + /* At least one header and one vp8 byte */ + if (G_UNLIKELY (size < 2)) return FALSE; + + out->size = size; + out->pts = GST_BUFFER_PTS (rtp->buffer); + out->end_of_frame = gst_rtp_buffer_get_marker (rtp); + out->picture_id = PICTURE_ID_NONE; + out->temporally_scaled = FALSE; + out->tl0picidx = 0; + out->tid = 0; + out->layer_sync = 0; + out->temporal_key_idx = 0; + out->hdrsize = 0; + +#define fail_if(val) if (G_UNLIKELY (val)) return FALSE; + /* Extended control bits present (X bit) */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &has_ext_ctrl_bits, 1)); + /* Reserved bit (R bit) */ + fail_if (!gst_bit_reader_skip (&br, 1)); + /* Non-reference frame (N bit) */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &out->is_non_ref_frame, 1)); + /* Start of VP8 partition (S bit) */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &out->part_start, 1)); + /* Reserved bit (R bit) */ + fail_if (!gst_bit_reader_skip (&br, 1)); + /* Partition index (PID bits) */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &out->part_idx, 3)); + + out->frame_start = (out->part_start && !out->part_idx); + + /* Check X optional header */ + if (has_ext_ctrl_bits) { + guint8 has_picture_id = 0; + guint8 has_tl0picidx = 0; + guint8 tid_set = 0; + guint8 keyidx_set = 0; + + /* Check I optional header */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &has_picture_id, 1)); + /* Check L optional header */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &has_tl0picidx, 1)); + /* Check T is set */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &tid_set, 1)); + /* Check K is set */ + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &keyidx_set, 1)); + /* Reserved bit (R bit) */ + fail_if (!gst_bit_reader_skip (&br, 4)); + + /* Stream is temporally scaled if L or T bits are set */ + out->temporally_scaled = (has_tl0picidx || tid_set); + + if (has_picture_id) { + guint8 is_ext_pic_id = 0; + fail_if (!gst_bit_reader_peek_bits_uint8 (&br, &is_ext_pic_id, 1)); + fail_if (!gst_bit_reader_get_bits_uint32 (&br, &out->picture_id, is_ext_pic_id ? 16 : 8)); + } + + if (has_tl0picidx) { + /* TL0PICIDX must be ignored unless T is set */ + if (tid_set) { + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &out->tl0picidx, 8)); + } else { + fail_if (!gst_bit_reader_skip (&br, 8)); + } + } + + if (tid_set || keyidx_set) { + /* TID and Y must be ignored unless T is set */ + if (tid_set) { + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &out->tid, 2)); + fail_if (!gst_bit_reader_peek_bits_uint8 (&br, &out->layer_sync, 1)); + } else { + fail_if (!gst_bit_reader_skip (&br, 3)); + } + /* KEYIDX must be ignored unless K is set */ + if (keyidx_set) { + fail_if (!gst_bit_reader_get_bits_uint8 (&br, &out->temporal_key_idx, 5)); + } else { + fail_if (!gst_bit_reader_skip (&br, 5)); + } + } + } +#undef fail_if + out->hdrsize = (guint8) ((gst_bit_reader_get_pos (&br) + 1) / 8); + return TRUE; +} + +typedef struct _GstVP8PFrameInfo +{ + gboolean is_keyframe; + guint profile; + guint width; + guint height; +} GstVP8PFrameInfo; + +static gboolean +gst_rtp_vp8_depay_parse_frame_descriptor (GstRtpVP8Depay * self, + GstVP8PFrameInfo * out) +{ + guint8 header[10]; + + if (gst_adapter_available (self->adapter) < 10) { + return FALSE; + } + + gst_adapter_copy (self->adapter, &header, 0, 10); + out->is_keyframe = !(header[0] & 0x01); + out->profile = (header[0] & 0x0e) >> 1; + out->width = GST_READ_UINT16_LE (header + 6) & 0x3fff; + out->height = GST_READ_UINT16_LE (header + 8) & 0x3fff; + return TRUE; +} + static void -gst_rtp_vp8_depay_init (GstRtpVP8Depay * self) +gst_rtp_vp8_depay_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) { - gst_rtp_base_depayload_set_aggregate_hdrext_enabled (GST_RTP_BASE_DEPAYLOAD - (self), TRUE); + GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY_CAST (object); + switch (prop_id) { + case PROP_WAIT_FOR_KEYFRAME: + self->wait_for_keyframe = g_value_get_boolean (value); + break; + case PROP_HIDE_PICTURE_ID_GAP: + self->hide_picture_id_gap = g_value_get_boolean (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} +static void +gst_rtp_vp8_depay_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY_CAST (object); + switch (prop_id) { + case PROP_WAIT_FOR_KEYFRAME: + g_value_set_boolean (value, self->wait_for_keyframe); + break; + case PROP_HIDE_PICTURE_ID_GAP: + g_value_set_boolean (value, self->hide_picture_id_gap); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_rtp_vp8_depay_init (GstRtpVP8Depay * self) +{ self->adapter = gst_adapter_new (); self->started = FALSE; self->wait_for_keyframe = DEFAULT_WAIT_FOR_KEYFRAME; - self->request_keyframe = DEFAULT_REQUEST_KEYFRAME; self->last_pushed_was_lost_event = FALSE; } @@ -114,30 +297,23 @@ gst_rtp_vp8_depay_class_init (GstRtpVP8DepayClass * gst_rtp_vp8_depay_class) object_class->set_property = gst_rtp_vp8_depay_set_property; object_class->get_property = gst_rtp_vp8_depay_get_property; + element_class->change_state = gst_rtp_vp8_depay_change_state; + + depay_class->process_rtp_packet = gst_rtp_vp8_depay_process; + depay_class->handle_event = gst_rtp_vp8_depay_handle_event; + depay_class->packet_lost = gst_rtp_vp8_depay_packet_lost; + g_object_class_install_property (object_class, PROP_WAIT_FOR_KEYFRAME, g_param_spec_boolean ("wait-for-keyframe", "Wait for Keyframe", "Wait for the next keyframe after packet loss", DEFAULT_WAIT_FOR_KEYFRAME, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - /** - * GstRtpVP8Depay:request-keyframe: - * - * Request new keyframe when packet loss is detected - * - * Since: 1.20 - */ - g_object_class_install_property (object_class, PROP_REQUEST_KEYFRAME, - g_param_spec_boolean ("request-keyframe", "Request Keyframe", - "Request new keyframe when packet loss is detected", - DEFAULT_REQUEST_KEYFRAME, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - element_class->change_state = gst_rtp_vp8_depay_change_state; - - depay_class->process_rtp_packet = gst_rtp_vp8_depay_process; - depay_class->handle_event = gst_rtp_vp8_depay_handle_event; - depay_class->packet_lost = gst_rtp_vp8_depay_packet_lost; + g_object_class_install_property (object_class, PROP_HIDE_PICTURE_ID_GAP, + g_param_spec_boolean ("hide-picture-id-gap", "Hide Picture ID Gap", + "Wether to trigger a key-unit request when there is a gap in " + "the picture ID", DEFAULT_HIDE_PICTURE_ID_GAP, + G_PARAM_CONSTRUCT | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); GST_DEBUG_CATEGORY_INIT (gst_rtp_vp8_depay_debug, "rtpvp8depay", 0, "VP8 Video RTP Depayloader"); @@ -158,64 +334,27 @@ gst_rtp_vp8_depay_dispose (GObject * object) G_OBJECT_CLASS (gst_rtp_vp8_depay_parent_class)->dispose (object); } -static void -gst_rtp_vp8_depay_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec) -{ - GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY (object); - - switch (prop_id) { - case PROP_WAIT_FOR_KEYFRAME: - self->wait_for_keyframe = g_value_get_boolean (value); - break; - case PROP_REQUEST_KEYFRAME: - self->request_keyframe = g_value_get_boolean (value); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} - -static void -gst_rtp_vp8_depay_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec) -{ - GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY (object); - - switch (prop_id) { - case PROP_WAIT_FOR_KEYFRAME: - g_value_set_boolean (value, self->wait_for_keyframe); - break; - case PROP_REQUEST_KEYFRAME: - g_value_set_boolean (value, self->request_keyframe); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} - static gint picture_id_compare (guint16 id0, guint16 id1) { guint shift = 16 - (IS_PICTURE_ID_15BITS (id1) ? 15 : 7); - id0 = id0 << shift; - id1 = id1 << shift; + id0 = (guint16) (id0 << shift); + id1 = (guint16) (id1 << shift); return ((gint16) (id1 - id0)) >> shift; } static void -send_last_lost_event (GstRtpVP8Depay * self) +send_last_lost_event (GstRtpVP8Depay * self, const gchar * reason) { if (self->last_lost_event) { - GST_ERROR_OBJECT (self, - "Sending the last stopped lost event: %" GST_PTR_FORMAT, - self->last_lost_event); + GST_DEBUG_OBJECT (self, + "Sending the last stopped lost event: %" GST_PTR_FORMAT + " reason \"%s\"", self->last_lost_event, reason ? reason : "None"); GST_RTP_BASE_DEPAYLOAD_CLASS (gst_rtp_vp8_depay_parent_class) ->packet_lost (GST_RTP_BASE_DEPAYLOAD_CAST (self), self->last_lost_event); - gst_event_replace (&self->last_lost_event, NULL); + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; self->last_pushed_was_lost_event = TRUE; } } @@ -234,7 +373,8 @@ send_new_lost_event (GstRtpVP8Depay * self, GstClockTime timestamp, event = gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM, gst_structure_new ("GstRTPPacketLost", "timestamp", G_TYPE_UINT64, timestamp, - "duration", G_TYPE_UINT64, G_GUINT64_CONSTANT (0), NULL)); + "duration", G_TYPE_UINT64, G_GUINT64_CONSTANT (0), + "no-packet-loss", G_TYPE_BOOLEAN, self->hide_picture_id_gap, NULL)); GST_DEBUG_OBJECT (self, "Pushing lost event " "(picids 0x%x 0x%x, reason \"%s\"): %" GST_PTR_FORMAT, @@ -244,257 +384,262 @@ send_new_lost_event (GstRtpVP8Depay * self, GstClockTime timestamp, ->packet_lost (GST_RTP_BASE_DEPAYLOAD_CAST (self), event); gst_event_unref (event); + self->last_pushed_was_lost_event = TRUE; +} + +static void +send_lost_event (GstRtpVP8Depay * self, GstClockTime timestamp, + guint picture_id, const gchar * reason) +{ + if (self->last_lost_event) { + send_last_lost_event (self, reason); + } else { + /* FIXME: Add property to control whether to send GAP events */ + send_new_lost_event (self, timestamp, picture_id, reason); + } +} + + +static void +send_keyframe_request (GstRtpVP8Depay * self) +{ + GST_DEBUG_OBJECT (self, "Sending keyframe request"); + if (!gst_pad_push_event (GST_RTP_BASE_DEPAYLOAD_SINKPAD (self), + gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, + TRUE, 0))) { + GST_ERROR_OBJECT (self, "Failed to push keyframe request"); + } +} + + +static void +drop_last_lost_event (GstRtpVP8Depay * self) +{ + if (self->last_lost_event) { + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; + } } static void -send_last_lost_event_if_needed (GstRtpVP8Depay * self, guint new_picture_id) +gst_rtp_vp8_depay_hadle_picture_id_gap (GstRtpVP8Depay * self, + guint new_picture_id, GstClockTime lost_event_timestamp) { + const gchar *reason = NULL; + gboolean fwd_last_lost_event = FALSE; + gboolean create_lost_event = FALSE; + gboolean gap_event_sent = FALSE; + if (self->last_picture_id == PICTURE_ID_NONE) return; + if (new_picture_id == PICTURE_ID_NONE) { + reason = "picture id does not exist"; + fwd_last_lost_event = TRUE; + } else if (IS_PICTURE_ID_15BITS (self->last_picture_id) && + !IS_PICTURE_ID_15BITS (new_picture_id)) { + reason = "picture id has less bits than before"; + fwd_last_lost_event = TRUE; + } else if (picture_id_compare ((guint16) self->last_picture_id, + (guint16) new_picture_id) != 1) { + reason = "picture id gap"; + fwd_last_lost_event = TRUE; + /* Only create a new one if we just didn't push a lost event */ + create_lost_event = self->last_pushed_was_lost_event == FALSE; + } + if (self->last_lost_event) { - gboolean send_lost_event = FALSE; - if (new_picture_id == PICTURE_ID_NONE) { - GST_DEBUG_OBJECT (self, "Dropping the last stopped lost event " - "(picture id does not exist): %" GST_PTR_FORMAT, - self->last_lost_event); - } else if (IS_PICTURE_ID_15BITS (self->last_picture_id) && - !IS_PICTURE_ID_15BITS (new_picture_id)) { - GST_DEBUG_OBJECT (self, "Dropping the last stopped lost event " - "(picture id has less bits than before): %" GST_PTR_FORMAT, - self->last_lost_event); - } else if (picture_id_compare (self->last_picture_id, new_picture_id) != 1) { - GstStructure *s = gst_event_writable_structure (self->last_lost_event); - - GST_DEBUG_OBJECT (self, "Sending the last stopped lost event " - "(gap in picture id %u %u): %" GST_PTR_FORMAT, - self->last_picture_id, new_picture_id, self->last_lost_event); - send_lost_event = TRUE; - /* Prevent rtpbasedepayload from dropping the event now - * that we have made sure the lost packet was not FEC */ - gst_structure_remove_field (s, "might-have-been-fec"); + if (fwd_last_lost_event) { + send_last_lost_event (self, reason); + gap_event_sent = TRUE; + } else { + drop_last_lost_event (self); } - if (send_lost_event) - GST_RTP_BASE_DEPAYLOAD_CLASS (gst_rtp_vp8_depay_parent_class) - ->packet_lost (GST_RTP_BASE_DEPAYLOAD_CAST (self), - self->last_lost_event); + } + + if (create_lost_event && !gap_event_sent) { + send_new_lost_event (self, lost_event_timestamp, new_picture_id, reason); + gap_event_sent = TRUE; + } - gst_event_replace (&self->last_lost_event, NULL); + if (gap_event_sent && self->waiting_for_keyframe) { + send_keyframe_request (self); } } -static GstBuffer * -gst_rtp_vp8_depay_process (GstRTPBaseDepayload * depay, GstRTPBuffer * rtp) + +static gboolean +gst_rtp_vp8_depay_reset_current_frame (GstRtpVP8Depay * self, + GstVP8PacketInfo * packet_info, const gchar * reason) { - GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY (depay); - GstBuffer *payload; - guint8 *data; - guint hdrsize = 1; - guint picture_id = PICTURE_ID_NONE; - guint size = gst_rtp_buffer_get_payload_len (rtp); - guint s_bit; - guint part_id; - gboolean frame_start; - gboolean sent_lost_event = FALSE; + self->started = FALSE; - if (G_UNLIKELY (GST_BUFFER_IS_DISCONT (rtp->buffer))) { - GST_DEBUG_OBJECT (self, "Discontinuity, flushing adapter"); - gst_adapter_clear (self->adapter); - self->started = FALSE; + if (!gst_adapter_available (self->adapter)) return FALSE; - if (self->wait_for_keyframe) - self->waiting_for_keyframe = TRUE; + GST_DEBUG_OBJECT (self, "%s, flushing adapter", reason); + gst_adapter_clear (self->adapter); - if (self->request_keyframe) - gst_pad_push_event (GST_RTP_BASE_DEPAYLOAD_SINKPAD (depay), - gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, - TRUE, 0)); + // Preventing for flooding with gap_events + if (!self->last_pushed_was_lost_event) { + send_lost_event (self, packet_info->pts, packet_info->picture_id, reason); } - /* At least one header and one vp8 byte */ - if (G_UNLIKELY (size < 2)) - goto too_small; - - data = gst_rtp_buffer_get_payload (rtp); - - s_bit = (data[0] >> 4) & 0x1; - part_id = (data[0] >> 0) & 0x7; + if (self->wait_for_keyframe) { + self->waiting_for_keyframe = TRUE; + } + if (self->waiting_for_keyframe) { + send_keyframe_request (self); + } + return TRUE; +} - /* Check X optional header */ - if ((data[0] & 0x80) != 0) { - hdrsize++; - /* Check I optional header */ - if ((data[1] & 0x80) != 0) { - if (G_UNLIKELY (size < 3)) - goto too_small; - hdrsize++; - /* Check for 16 bits PictureID */ - picture_id = data[2]; - if ((data[2] & 0x80) != 0) { - if (G_UNLIKELY (size < 4)) - goto too_small; - hdrsize++; - picture_id = (picture_id << 8) | data[3]; - } +static GstBuffer * +gst_rtp_vp8_depay_get_frame (GstRtpVP8Depay * self, + const GstVP8PacketInfo * packet_info, const GstVP8PFrameInfo * frame_info) +{ + /* mark keyframes */ + GstBuffer *out = gst_adapter_take_buffer (self->adapter, + gst_adapter_available (self->adapter)); + + out = gst_buffer_make_writable (out); + + /* Filter away all metas that are not sensible to copy */ + gst_rtp_drop_non_video_meta (self, out); + + GstCustomMeta *meta = gst_buffer_add_custom_meta (out, "GstVP8Meta"); + GstStructure *s = gst_custom_meta_get_structure (meta); + gst_structure_set (s, + "use-temporal-scaling", G_TYPE_BOOLEAN, packet_info->temporally_scaled, + "layer-sync", G_TYPE_BOOLEAN, packet_info->layer_sync, + "layer-id", G_TYPE_UINT, packet_info->tid, + "tl0picidx", G_TYPE_UINT, packet_info->tl0picidx, NULL); + + if (frame_info->is_keyframe) { + GST_BUFFER_FLAG_UNSET (out, GST_BUFFER_FLAG_DELTA_UNIT); + GST_DEBUG_OBJECT (self, "Processed keyframe"); + + if (G_UNLIKELY (self->last_width != frame_info->width || + self->last_height != frame_info->height || + self->last_profile != frame_info->profile)) { + gchar profile_str[3]; + GstCaps *srccaps; + + snprintf (profile_str, 3, "%u", frame_info->profile); + srccaps = gst_caps_new_simple ("video/x-vp8", + "framerate", GST_TYPE_FRACTION, 0, 1, + "height", G_TYPE_INT, frame_info->height, + "width", G_TYPE_INT, frame_info->width, + "profile", G_TYPE_STRING, profile_str, NULL); + + gst_pad_set_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (self), srccaps); + gst_caps_unref (srccaps); + + self->last_width = frame_info->width; + self->last_height = frame_info->height; + self->last_profile = frame_info->profile; } - /* Check L optional header */ - if ((data[1] & 0x40) != 0) - hdrsize++; - /* Check T or K optional headers */ - if ((data[1] & 0x20) != 0 || (data[1] & 0x10) != 0) - hdrsize++; - } - GST_LOG_OBJECT (depay, - "hdrsize %u, size %u, picture id 0x%x, s %u, part_id %u", hdrsize, size, - picture_id, s_bit, part_id); - if (G_UNLIKELY (hdrsize >= size)) - goto too_small; - - frame_start = (s_bit == 1) && (part_id == 0); - if (frame_start) { - if (G_UNLIKELY (self->started)) { - GST_DEBUG_OBJECT (depay, "Incomplete frame, flushing adapter"); - /* keep the current buffer because it may still be used later */ - gst_rtp_base_depayload_flush (depay, TRUE); - gst_adapter_clear (self->adapter); - self->started = FALSE; - - if (self->wait_for_keyframe) - self->waiting_for_keyframe = TRUE; - if (self->request_keyframe) - gst_pad_push_event (GST_RTP_BASE_DEPAYLOAD_SINKPAD (depay), - gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, - TRUE, 0)); - - send_new_lost_event (self, GST_BUFFER_PTS (rtp->buffer), picture_id, - "Incomplete frame detected"); - sent_lost_event = TRUE; + self->waiting_for_keyframe = FALSE; + } else { + GST_BUFFER_FLAG_SET (out, GST_BUFFER_FLAG_DELTA_UNIT); + GST_DEBUG_OBJECT (self, "Processed interframe"); + + if (self->waiting_for_keyframe) { + gst_buffer_unref (out); + out = NULL; + GST_INFO_OBJECT (self, "Dropping inter-frame before intra-frame"); + send_keyframe_request (self); } } - if (!self->started) { - if (G_UNLIKELY (!frame_start)) { - GST_DEBUG_OBJECT (depay, - "The frame is missing the first packet, ignoring the packet"); - if (self->stop_lost_events && !sent_lost_event) { - send_last_lost_event (self); - self->stop_lost_events = FALSE; - } + return out; +} - if (self->wait_for_keyframe) - self->waiting_for_keyframe = TRUE; - if (self->request_keyframe) - gst_pad_push_event (GST_RTP_BASE_DEPAYLOAD_SINKPAD (depay), - gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, - TRUE, 0)); +static GstBuffer * +gst_rtp_vp8_depay_process (GstRTPBaseDepayload * depay, GstRTPBuffer * rtp) +{ + GstBuffer *out = NULL; + GstVP8PacketInfo packet_info; + GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY_CAST (depay); + + if (G_UNLIKELY (!gst_rtp_vp8_depay_parse_header (rtp, &packet_info))) { + gst_rtp_vp8_depay_reset_current_frame (self, &packet_info, + "Invalid rtp packet detected"); + return NULL; + } - goto done; - } + GST_LOG_OBJECT (depay, + "hdrsize %u, size %u, picture id 0x%x, s %u, part_id %u", + packet_info.hdrsize, packet_info.size, packet_info.picture_id, + packet_info.part_start, packet_info.part_idx); + + if (G_UNLIKELY (GST_BUFFER_IS_DISCONT (rtp->buffer))) { + gst_rtp_vp8_depay_reset_current_frame (self, &packet_info, + "Discontinuity detected"); + } + + if (G_UNLIKELY (packet_info.frame_start == self->started)) { + // We either didn't completed previous frame + // or didn't start next frame + gst_rtp_vp8_depay_reset_current_frame (self, &packet_info, + "Incomplete frame detected"); + } + if (packet_info.frame_start) { GST_LOG_OBJECT (depay, "Found the start of the frame"); - if (self->stop_lost_events && !sent_lost_event) { - send_last_lost_event_if_needed (self, picture_id); - self->stop_lost_events = FALSE; - } + gst_rtp_vp8_depay_hadle_picture_id_gap (self, packet_info.picture_id, + packet_info.pts); self->started = TRUE; + self->stop_lost_events = FALSE; + self->last_picture_id = packet_info.picture_id; + + } else if (self->started) { + // PictureID gap in a middle of the frame + if (self->last_picture_id != packet_info.picture_id) + gst_rtp_vp8_depay_reset_current_frame (self, &packet_info, + "picture id gap"); } - payload = gst_rtp_buffer_get_payload_subbuffer (rtp, hdrsize, -1); - gst_adapter_push (self->adapter, payload); - self->last_picture_id = picture_id; - - /* Marker indicates that it was the last rtp packet for this frame */ - if (gst_rtp_buffer_get_marker (rtp)) { - GstBuffer *out; - guint8 header[10]; - - GST_LOG_OBJECT (depay, - "Found the end of the frame (%" G_GSIZE_FORMAT " bytes)", - gst_adapter_available (self->adapter)); - if (gst_adapter_available (self->adapter) < 10) - goto too_small; - gst_adapter_copy (self->adapter, &header, 0, 10); - - out = gst_adapter_take_buffer (self->adapter, - gst_adapter_available (self->adapter)); - - self->started = FALSE; - - /* mark keyframes */ - out = gst_buffer_make_writable (out); - /* Filter away all metas that are not sensible to copy */ - gst_rtp_drop_non_video_meta (self, out); - if ((header[0] & 0x01)) { - GST_BUFFER_FLAG_SET (out, GST_BUFFER_FLAG_DELTA_UNIT); - - if (self->waiting_for_keyframe) { - gst_rtp_base_depayload_flush (depay, FALSE); - gst_buffer_unref (out); - out = NULL; - GST_INFO_OBJECT (self, "Dropping inter-frame before intra-frame"); - gst_pad_push_event (GST_RTP_BASE_DEPAYLOAD_SINKPAD (depay), - gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, - TRUE, 0)); - } - } else { - guint profile, width, height; + if (self->started) { + /* Store rtp payload data in adapter */ + gst_adapter_push (self->adapter, gst_rtp_buffer_get_payload_subbuffer (rtp, + packet_info.hdrsize, -1)); - GST_BUFFER_FLAG_UNSET (out, GST_BUFFER_FLAG_DELTA_UNIT); - GST_DEBUG_OBJECT (self, "Processed keyframe"); + /* Marker indicates that it was the last rtp packet for this frame */ + if (packet_info.end_of_frame) { + GstVP8PFrameInfo frame_info; - profile = (header[0] & 0x0e) >> 1; - width = GST_READ_UINT16_LE (header + 6) & 0x3fff; - height = GST_READ_UINT16_LE (header + 8) & 0x3fff; + GST_LOG_OBJECT (depay, + "Found the end of the frame (%" G_GSIZE_FORMAT " bytes)", + gst_adapter_available (self->adapter)); - if (G_UNLIKELY (self->last_width != width || - self->last_height != height || self->last_profile != profile)) { - gchar profile_str[3]; - GstCaps *srccaps; + if (gst_rtp_vp8_depay_parse_frame_descriptor (self, &frame_info)) { + out = gst_rtp_vp8_depay_get_frame (self, &packet_info, &frame_info); - snprintf (profile_str, 3, "%u", profile); - srccaps = gst_caps_new_simple ("video/x-vp8", - "framerate", GST_TYPE_FRACTION, 0, 1, - "height", G_TYPE_INT, height, - "width", G_TYPE_INT, width, - "profile", G_TYPE_STRING, profile_str, NULL); + if (packet_info.picture_id != PICTURE_ID_NONE) + self->stop_lost_events = TRUE; - gst_pad_set_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (depay), srccaps); - gst_caps_unref (srccaps); - - self->last_width = width; - self->last_height = height; - self->last_profile = profile; + self->last_pushed_was_lost_event = FALSE; + self->started = FALSE; + } else { + gst_rtp_vp8_depay_reset_current_frame (self, &packet_info, + "Invalid rtp packet detected"); } - self->waiting_for_keyframe = FALSE; } - - if (picture_id != PICTURE_ID_NONE) - self->stop_lost_events = TRUE; - - self->last_pushed_was_lost_event = FALSE; - - return out; + } else { + // Wating for start of the new frame + GST_DEBUG_OBJECT (depay, + "The frame is missing the first packet, ignoring the packet"); } -done: - gst_rtp_base_depayload_dropped (depay); - return NULL; - -too_small: - GST_DEBUG_OBJECT (self, "Invalid rtp packet (too small), ignoring"); - gst_rtp_base_depayload_flush (depay, FALSE); - gst_adapter_clear (self->adapter); - self->started = FALSE; - - goto done; + return out; } static GstStateChangeReturn gst_rtp_vp8_depay_change_state (GstElement * element, GstStateChange transition) { - GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY (element); + GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY_CAST (element); switch (transition) { case GST_STATE_CHANGE_READY_TO_PAUSED: @@ -504,7 +649,10 @@ gst_rtp_vp8_depay_change_state (GstElement * element, GstStateChange transition) self->waiting_for_keyframe = TRUE; self->caps_sent = FALSE; self->last_picture_id = PICTURE_ID_NONE; - gst_event_replace (&self->last_lost_event, NULL); + if (self->last_lost_event) { + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; + } self->stop_lost_events = FALSE; break; default: @@ -519,7 +667,7 @@ gst_rtp_vp8_depay_change_state (GstElement * element, GstStateChange transition) static gboolean gst_rtp_vp8_depay_handle_event (GstRTPBaseDepayload * depay, GstEvent * event) { - GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY (depay); + GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY_CAST (depay); switch (GST_EVENT_TYPE (event)) { case GST_EVENT_FLUSH_STOP: @@ -527,7 +675,10 @@ gst_rtp_vp8_depay_handle_event (GstRTPBaseDepayload * depay, GstEvent * event) self->last_height = -1; self->last_width = -1; self->last_picture_id = PICTURE_ID_NONE; - gst_event_replace (&self->last_lost_event, NULL); + if (self->last_lost_event) { + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; + } self->stop_lost_events = FALSE; break; default: @@ -542,45 +693,18 @@ gst_rtp_vp8_depay_handle_event (GstRTPBaseDepayload * depay, GstEvent * event) static gboolean gst_rtp_vp8_depay_packet_lost (GstRTPBaseDepayload * depay, GstEvent * event) { - GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY (depay); - const GstStructure *s; - gboolean might_have_been_fec; - gboolean unref_event = FALSE; - gboolean ret; - - s = gst_event_get_structure (event); - + GstRtpVP8Depay *self = GST_RTP_VP8_DEPAY_CAST (depay); if (self->stop_lost_events) { - if (gst_structure_get_boolean (s, "might-have-been-fec", - &might_have_been_fec) - && might_have_been_fec) { - GST_DEBUG_OBJECT (depay, "Stopping lost event %" GST_PTR_FORMAT, event); - gst_event_replace (&self->last_lost_event, event); - return TRUE; - } - } else if (self->last_picture_id != PICTURE_ID_NONE) { - GstStructure *s; - - if (!gst_event_is_writable (event)) { - event = gst_event_copy (event); - unref_event = TRUE; - } - - s = gst_event_writable_structure (event); - - /* We are currently processing a picture, let's make sure the - * base depayloader doesn't drop this lost event */ - gst_structure_remove_field (s, "might-have-been-fec"); + GST_DEBUG_OBJECT (depay, "Stopping lost event %" GST_PTR_FORMAT, event); + if (self->last_lost_event) + gst_event_unref (self->last_lost_event); + self->last_lost_event = gst_event_ref (event); + return TRUE; } self->last_pushed_was_lost_event = TRUE; - ret = + return GST_RTP_BASE_DEPAYLOAD_CLASS (gst_rtp_vp8_depay_parent_class)->packet_lost (depay, event); - - if (unref_event) - gst_event_unref (event); - - return ret; -} +} \ No newline at end of file diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.h b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.h index 76be52adc7b..388ce88cb2b 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.h +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8depay.h @@ -24,7 +24,6 @@ #include G_BEGIN_DECLS - #define GST_TYPE_RTP_VP8_DEPAY \ (gst_rtp_vp8_depay_get_type()) #define GST_RTP_VP8_DEPAY(obj) \ @@ -39,6 +38,7 @@ G_BEGIN_DECLS #define GST_RTP_VP8_DEPAY_GET_CLASS(obj) \ (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_RTP_VP8_DEPAY, \ GstRtpVP8DepayClass)) +#define GST_RTP_VP8_DEPAY_CAST(obj) ((GstRtpVP8Depay *)(obj)) typedef struct _GstRtpVP8Depay GstRtpVP8Depay; typedef struct _GstRtpVP8DepayClass GstRtpVP8DepayClass; @@ -55,23 +55,18 @@ struct _GstRtpVP8Depay gboolean started; gboolean caps_sent; - /* In between pictures, we might store GstRTPPacketLost events instead - * of forwarding them immediately, we check upon reception of a new - * picture id whether a gap was introduced, in which case we do forward - * the event. This is to avoid forwarding spurious lost events for FEC - * packets. - */ gboolean stop_lost_events; GstEvent *last_lost_event; gboolean waiting_for_keyframe; + gboolean last_pushed_was_lost_event; gint last_profile; gint last_width; gint last_height; guint last_picture_id; + /* properties */ gboolean wait_for_keyframe; - gboolean request_keyframe; - gboolean last_pushed_was_lost_event; + gboolean hide_picture_id_gap; }; GType gst_rtp_vp8_depay_get_type (void); diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c index c216a125cfa..1b8118ccbac 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.c @@ -41,13 +41,15 @@ GST_DEBUG_CATEGORY_STATIC (gst_rtp_vp8_pay_debug); #define DEFAULT_PICTURE_ID_MODE VP8_PAY_NO_PICTURE_ID #define DEFAULT_PICTURE_ID_OFFSET (-1) +#define DEFAULT_PARSE_FRAMES TRUE enum { PROP_0, PROP_PICTURE_ID, PROP_PICTURE_ID_MODE, - PROP_PICTURE_ID_OFFSET + PROP_PICTURE_ID_OFFSET, + PROP_PARSE_FRAMES, }; #define GST_TYPE_RTP_VP8_PAY_PICTURE_ID_MODE (gst_rtp_vp8_pay_picture_id_mode_get_type()) @@ -193,6 +195,12 @@ gst_rtp_vp8_pay_class_init (GstRtpVP8PayClass * gst_rtp_vp8_pay_class) GST_TYPE_RTP_VP8_PAY_PICTURE_ID_MODE, DEFAULT_PICTURE_ID_MODE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PARSE_FRAMES, + g_param_spec_boolean ("parse-frames", "Parse frames", + "Whether to parse/validate frame bitstreams", + DEFAULT_PARSE_FRAMES, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT_ONLY)); + /** * rtpvp8pay:picture-id-offset: * @@ -240,6 +248,9 @@ gst_rtp_vp8_pay_set_property (GObject * object, rtpvp8pay->picture_id_offset = g_value_get_int (value); gst_rtp_vp8_pay_picture_id_reset (rtpvp8pay); break; + case PROP_PARSE_FRAMES: + rtpvp8pay->parse_frames = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -262,6 +273,9 @@ gst_rtp_vp8_pay_get_property (GObject * object, case PROP_PICTURE_ID_OFFSET: g_value_set_int (value, rtpvp8pay->picture_id_offset); break; + case PROP_PARSE_FRAMES: + g_value_set_boolean (value, rtpvp8pay->parse_frames); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -297,7 +311,20 @@ gst_rtp_vp8_pay_parse_frame (GstRtpVP8Pay * self, GstBuffer * buffer, gst_bit_reader_init (&reader, data, size); - self->is_keyframe = keyframe = ((data[0] & 0x1) == 0); + /* from RFC 6386 Section 9.1: Uncompressed data chunk + * + * 0 1 2 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |F|Vers.|S| Partition size | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * Frame type: 0 key-frame, 1 inter-frame + * Version (0-3) + * Show_frame + * First partition size + * + */ + keyframe = ((data[0] & 0x1) == 0); version = (data[0] >> 1) & 0x7; if (G_UNLIKELY (version > 3)) { @@ -316,6 +343,16 @@ gst_rtp_vp8_pay_parse_frame (GstRtpVP8Pay * self, GstBuffer * buffer, goto error; if (keyframe) { + /* + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | 0x9d012a | Scale & Width | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | | Scale & Height | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + */ /* check start tag: 0x9d 0x01 0x2a */ if (!gst_bit_reader_get_bits_uint8 (&reader, &tmp8, 8) || tmp8 != 0x9d) goto error; @@ -326,7 +363,9 @@ gst_rtp_vp8_pay_parse_frame (GstRtpVP8Pay * self, GstBuffer * buffer, if (!gst_bit_reader_get_bits_uint8 (&reader, &tmp8, 8) || tmp8 != 0x2a) goto error; - /* Skip horizontal size code (16 bits) vertical size code (16 bits) */ + /* Skip horizontal size code (16 bits) vertical size code (16 bits) + * (2 bits horizonal scaling << 14 | Width) + * (2 bits vertical scaling << 14 | Height) */ if (!gst_bit_reader_skip (&reader, 32)) goto error; } @@ -410,16 +449,18 @@ gst_rtp_vp8_pay_parse_frame (GstRtpVP8Pay * self, GstBuffer * buffer, pdata = data + partition0_size; /* Set up mapping */ + guint partition_size[9]; + self->n_partitions = partitions + 1; self->partition_offset[0] = 0; - self->partition_size[0] = partition0_size + (partitions - 1) * 3; + partition_size[0] = partition0_size + (partitions - 1) * 3; - self->partition_offset[1] = self->partition_size[0]; + self->partition_offset[1] = partition_size[0]; for (i = 1; i < partitions; i++) { guint psize = (pdata[2] << 16 | pdata[1] << 8 | pdata[0]); pdata += 3; - self->partition_size[i] = psize; + partition_size[i] = psize; self->partition_offset[i + 1] = self->partition_offset[i] + psize; } @@ -428,7 +469,7 @@ gst_rtp_vp8_pay_parse_frame (GstRtpVP8Pay * self, GstBuffer * buffer, if (self->partition_offset[i] >= size) goto error; - self->partition_size[i] = size - self->partition_offset[i]; + partition_size[i] = size - self->partition_offset[i]; self->partition_offset[i + 1] = size; @@ -627,10 +668,12 @@ gst_rtp_vp8_payload_next (GstRtpVP8Pay * self, GstBufferList * list, if (available > remaining) available = remaining; - if (meta) { + if (meta || !self->parse_frames) { /* If meta is present, then we have no partition offset information, * so always emit PID 0 and set the start bit for the first packet - * of a frame only (c.f. RFC7741 $4.4) + * of a frame only (c.f. RFC7741 $4.4). Or, when we want to avoid parsing + * the bitstream as currently partition information is only present in + * the VP8 frame itself. */ partition = 0; start = (offset == 0); @@ -675,10 +718,12 @@ gst_rtp_vp8_pay_handle_buffer (GstRTPBasePayload * payload, GstBuffer * buffer) meta = gst_buffer_get_custom_meta (buffer, "GstVP8Meta"); delta_unit = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT); - if (G_UNLIKELY (!gst_rtp_vp8_pay_parse_frame (self, buffer, size))) { - GST_ELEMENT_ERROR (self, STREAM, ENCODE, (NULL), - ("Failed to parse VP8 frame")); - return GST_FLOW_ERROR; + if (self->parse_frames) { + if (G_UNLIKELY (!gst_rtp_vp8_pay_parse_frame (self, buffer, size))) { + GST_ELEMENT_ERROR (self, STREAM, ENCODE, (NULL), + ("Failed to parse VP8 frame")); + return GST_FLOW_ERROR; + } } if (meta) { diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.h b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.h index 748bec16501..132fc0cdfe4 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.h +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp8pay.h @@ -1,3 +1,4 @@ +/* /* * gstrtpvp8pay.h - Header for GstRtpVP8Pay * Copyright (C) 2011 Sjoerd Simons @@ -55,9 +56,10 @@ struct _GstRtpVP8PayClass struct _GstRtpVP8Pay { GstRTPBasePayload parent; + gboolean parse_frames; gboolean is_keyframe; gint n_partitions; - /* Treat frame header & tag & partition size block as the first partition, + /* Treat frame header & tag as the first partition, * folowed by max. 8 data partitions. last offset is the end of the buffer */ guint partition_offset[10]; guint partition_size[9]; diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.c index 5c5f71c76c2..f1ae8461286 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.c @@ -35,10 +35,6 @@ GST_DEBUG_CATEGORY_STATIC (gst_rtp_vp9_depay_debug); #define GST_CAT_DEFAULT gst_rtp_vp9_depay_debug static void gst_rtp_vp9_depay_dispose (GObject * object); -static void gst_rtp_vp9_depay_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec); -static void gst_rtp_vp9_depay_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec); static GstBuffer *gst_rtp_vp9_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp); static GstStateChangeReturn gst_rtp_vp9_depay_change_state (GstElement * @@ -67,30 +63,55 @@ GST_STATIC_PAD_TEMPLATE ("sink", "media = (string) \"video\"," "encoding-name = (string) { \"VP9\", \"VP9-DRAFT-IETF-01\" }")); -#define DEFAULT_WAIT_FOR_KEYFRAME FALSE -#define DEFAULT_REQUEST_KEYFRAME FALSE +#define PICTURE_ID_NONE (UINT_MAX) +#define IS_PICTURE_ID_15BITS(pid) (((guint)(pid) & 0x8000) != 0) +#define PICTURE_ID_WITHOUT_MBIT(pid) ((pid) & 0x7fff) + +#define DEFAULT_HIDE_PICTURE_ID_GAP FALSE enum { PROP_0, - PROP_WAIT_FOR_KEYFRAME, - PROP_REQUEST_KEYFRAME, + PROP_HIDE_PICTURE_ID_GAP, }; -#define PICTURE_ID_NONE (UINT_MAX) -#define IS_PICTURE_ID_15BITS(pid) (((guint)(pid) & 0x8000) != 0) +static void +gst_rtp_vp9_depay_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstRtpVP9Depay *self = GST_RTP_VP9_DEPAY_CAST (object); + switch (prop_id) { + case PROP_HIDE_PICTURE_ID_GAP: + self->hide_picture_id_gap = g_value_get_boolean (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} static void -gst_rtp_vp9_depay_init (GstRtpVP9Depay * self) +gst_rtp_vp9_depay_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) { - gst_rtp_base_depayload_set_aggregate_hdrext_enabled (GST_RTP_BASE_DEPAYLOAD - (self), TRUE); + GstRtpVP9Depay *self = GST_RTP_VP9_DEPAY_CAST (object); + switch (prop_id) { + case PROP_HIDE_PICTURE_ID_GAP: + g_value_set_boolean (value, self->hide_picture_id_gap); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} +static void +gst_rtp_vp9_depay_init (GstRtpVP9Depay * self) +{ self->adapter = gst_adapter_new (); self->started = FALSE; self->inter_picture = FALSE; - self->wait_for_keyframe = DEFAULT_WAIT_FOR_KEYFRAME; - self->request_keyframe = DEFAULT_REQUEST_KEYFRAME; + self->last_pushed_was_lost_event = FALSE; } static void @@ -111,35 +132,9 @@ gst_rtp_vp9_depay_class_init (GstRtpVP9DepayClass * gst_rtp_vp9_depay_class) "Codec/Depayloader/Network/RTP", "Extracts VP9 video from RTP packets)", "Stian Selnes "); - object_class->dispose = gst_rtp_vp9_depay_dispose; object_class->set_property = gst_rtp_vp9_depay_set_property; object_class->get_property = gst_rtp_vp9_depay_get_property; - - /** - * GstRtpVP9Depay:wait-for-keyframe: - * - * Wait for the next keyframe after packet loss - * - * Since: 1.22 - */ - g_object_class_install_property (object_class, PROP_WAIT_FOR_KEYFRAME, - g_param_spec_boolean ("wait-for-keyframe", "Wait for Keyframe", - "Wait for the next keyframe after packet loss", - DEFAULT_WAIT_FOR_KEYFRAME, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - - /** - * GstRtpVP9Depay:request-keyframe: - * - * Request new keyframe when packet loss is detected - * - * Since: 1.22 - */ - g_object_class_install_property (object_class, PROP_REQUEST_KEYFRAME, - g_param_spec_boolean ("request-keyframe", "Request Keyframe", - "Request new keyframe when packet loss is detected", - DEFAULT_REQUEST_KEYFRAME, - G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + object_class->dispose = gst_rtp_vp9_depay_dispose; element_class->change_state = gst_rtp_vp9_depay_change_state; @@ -147,6 +142,12 @@ gst_rtp_vp9_depay_class_init (GstRtpVP9DepayClass * gst_rtp_vp9_depay_class) depay_class->handle_event = gst_rtp_vp9_depay_handle_event; depay_class->packet_lost = gst_rtp_vp9_depay_packet_lost; + g_object_class_install_property (object_class, PROP_HIDE_PICTURE_ID_GAP, + g_param_spec_boolean ("hide-picture-id-gap", "Hide Picture ID Gap", + "Wether to trigger a key-unit request when there is a gap in " + "the picture ID", DEFAULT_HIDE_PICTURE_ID_GAP, + G_PARAM_CONSTRUCT | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + GST_DEBUG_CATEGORY_INIT (gst_rtp_vp9_depay_debug, "rtpvp9depay", 0, "VP9 Video RTP Depayloader"); } @@ -166,45 +167,6 @@ gst_rtp_vp9_depay_dispose (GObject * object) G_OBJECT_CLASS (gst_rtp_vp9_depay_parent_class)->dispose (object); } -static void -gst_rtp_vp9_depay_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec) -{ - GstRtpVP9Depay *self = GST_RTP_VP9_DEPAY (object); - - switch (prop_id) { - case PROP_WAIT_FOR_KEYFRAME: - self->wait_for_keyframe = g_value_get_boolean (value); - break; - case PROP_REQUEST_KEYFRAME: - self->request_keyframe = g_value_get_boolean (value); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} - -static void -gst_rtp_vp9_depay_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec) -{ - GstRtpVP9Depay *self = GST_RTP_VP9_DEPAY (object); - - switch (prop_id) { - case PROP_WAIT_FOR_KEYFRAME: - g_value_set_boolean (value, self->wait_for_keyframe); - break; - case PROP_REQUEST_KEYFRAME: - g_value_set_boolean (value, self->request_keyframe); - break; - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} - - static gint picture_id_compare (guint16 id0, guint16 id1) { @@ -224,46 +186,86 @@ send_last_lost_event (GstRtpVP9Depay * self) GST_RTP_BASE_DEPAYLOAD_CLASS (gst_rtp_vp9_depay_parent_class) ->packet_lost (GST_RTP_BASE_DEPAYLOAD_CAST (self), self->last_lost_event); - gst_event_replace (&self->last_lost_event, NULL); + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; + self->last_pushed_was_lost_event = TRUE; } } static void -send_last_lost_event_if_needed (GstRtpVP9Depay * self, guint new_picture_id) +send_new_lost_event (GstRtpVP9Depay * self, GstClockTime timestamp, + guint new_picture_id, const gchar * reason) { + GstEvent *event; + + if (!GST_CLOCK_TIME_IS_VALID (timestamp)) { + GST_WARNING_OBJECT (self, + "Can't create lost event with invalid timestmap"); + return; + } + + event = gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM, + gst_structure_new ("GstRTPPacketLost", + "timestamp", G_TYPE_UINT64, timestamp, + "duration", G_TYPE_UINT64, 0, + "no-packet-loss", G_TYPE_BOOLEAN, self->hide_picture_id_gap, + NULL)); + + GST_DEBUG_OBJECT (self, "Pushing lost event " + "(picids 0x%x 0x%x, reason \"%s\"): %" GST_PTR_FORMAT, + self->last_picture_id, new_picture_id, reason, event); + + GST_RTP_BASE_DEPAYLOAD_CLASS (gst_rtp_vp9_depay_parent_class) + ->packet_lost (GST_RTP_BASE_DEPAYLOAD_CAST (self), event); + + gst_event_unref (event); +} + +static void +send_lost_event_if_needed (GstRtpVP9Depay * self, guint new_picture_id, + GstClockTime lost_event_timestamp) +{ + const gchar *reason = NULL; + gboolean fwd_last_lost_event = FALSE; + gboolean create_lost_event = FALSE; + if (self->last_picture_id == PICTURE_ID_NONE || self->last_picture_id == new_picture_id) return; + if (new_picture_id == PICTURE_ID_NONE) { + reason = "picture id does not exist"; + fwd_last_lost_event = TRUE; + } else if (IS_PICTURE_ID_15BITS (self->last_picture_id) && + !IS_PICTURE_ID_15BITS (new_picture_id)) { + reason = "picture id has less bits than before"; + fwd_last_lost_event = TRUE; + } else if (picture_id_compare (self->last_picture_id, new_picture_id) != 1) { + reason = "picture id gap"; + fwd_last_lost_event = TRUE; + /* Only create a new one if we just didn't push a lost event */ + create_lost_event = self->last_pushed_was_lost_event == FALSE; + } + if (self->last_lost_event) { - gboolean send_lost_event = FALSE; - if (new_picture_id == PICTURE_ID_NONE) { - GST_DEBUG_OBJECT (self, "Dropping the last stopped lost event " - "(picture id does not exist): %" GST_PTR_FORMAT, - self->last_lost_event); - } else if (IS_PICTURE_ID_15BITS (self->last_picture_id) && - !IS_PICTURE_ID_15BITS (new_picture_id)) { - GST_DEBUG_OBJECT (self, "Dropping the last stopped lost event " - "(picture id has less bits than before): %" GST_PTR_FORMAT, - self->last_lost_event); - } else if (picture_id_compare (self->last_picture_id, new_picture_id) != 1) { - GstStructure *s = gst_event_writable_structure (self->last_lost_event); - - GST_DEBUG_OBJECT (self, "Sending the last stopped lost event " - "(gap in picture id %u %u): %" GST_PTR_FORMAT, - self->last_picture_id, new_picture_id, self->last_lost_event); - send_lost_event = TRUE; - /* Prevent rtpbasedepayload from dropping the event now - * that we have made sure the lost packet was not FEC */ - gst_structure_remove_field (s, "might-have-been-fec"); - } - if (send_lost_event) + if (fwd_last_lost_event) { + GST_DEBUG_OBJECT (self, "Forwarding lost event " + "(picids 0x%x 0x%x, reason \"%s\"): %" GST_PTR_FORMAT, + self->last_picture_id, new_picture_id, reason, self->last_lost_event); GST_RTP_BASE_DEPAYLOAD_CLASS (gst_rtp_vp9_depay_parent_class) ->packet_lost (GST_RTP_BASE_DEPAYLOAD_CAST (self), self->last_lost_event); - gst_event_replace (&self->last_lost_event, NULL); + // If we forward last received lost event, there is no need + // to create another one + create_lost_event = FALSE; + } + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; } + + if (create_lost_event) + send_new_lost_event (self, lost_event_timestamp, new_picture_id, reason); } static GstBuffer * @@ -278,13 +280,12 @@ gst_rtp_vp9_depay_process (GstRTPBaseDepayload * depay, GstRTPBuffer * rtp) guint picture_id = PICTURE_ID_NONE; gboolean i_bit, p_bit, l_bit, f_bit, b_bit, e_bit, v_bit, d_bit = 0; gboolean is_start_of_picture; - gboolean flushed_adapter = FALSE; + gboolean sent_lost_event = FALSE; if (G_UNLIKELY (GST_BUFFER_IS_DISCONT (rtp->buffer))) { GST_LOG_OBJECT (self, "Discontinuity, flushing adapter"); gst_adapter_clear (self->adapter); self->started = FALSE; - flushed_adapter = TRUE; } size = gst_rtp_buffer_get_payload_len (rtp); @@ -424,38 +425,22 @@ gst_rtp_vp9_depay_process (GstRTPBaseDepayload * depay, GstRTPBuffer * rtp) if (is_start_of_picture) { if (G_UNLIKELY (self->started)) { GST_DEBUG_OBJECT (depay, "Incomplete frame, flushing adapter"); - /* keep the current buffer because it may still be used later */ - gst_rtp_base_depayload_flush (depay, TRUE); gst_adapter_clear (self->adapter); self->started = FALSE; - flushed_adapter = TRUE; + + /* FIXME: Add property to control whether to send GAP events */ + send_new_lost_event (self, GST_BUFFER_PTS (rtp->buffer), picture_id, + "Incomplete frame detected"); + sent_lost_event = TRUE; } } if (G_UNLIKELY (!self->started)) { - self->inter_picture = FALSE; - - /* We have flushed the adapter and this packet does not - * start a keyframe, request one if needed */ - if (flushed_adapter && (!b_bit || p_bit)) { - if (self->wait_for_keyframe) { - GST_DEBUG_OBJECT (self, "Waiting for keyframe after flushing adapter"); - self->waiting_for_keyframe = TRUE; - } - - if (self->request_keyframe) { - GST_DEBUG_OBJECT (self, "Requesting keyframe after flushing adapter"); - gst_pad_push_event (GST_RTP_BASE_DEPAYLOAD_SINKPAD (depay), - gst_video_event_new_upstream_force_key_unit (GST_CLOCK_TIME_NONE, - TRUE, 0)); - } - } - /* Check if this is the start of a VP9 layer frame, otherwise bail */ if (!b_bit) { GST_DEBUG_OBJECT (depay, "The layer is missing the first packets, ignoring the packet"); - if (self->stop_lost_events) { + if (self->stop_lost_events && !sent_lost_event) { send_last_lost_event (self); self->stop_lost_events = FALSE; } @@ -463,11 +448,11 @@ gst_rtp_vp9_depay_process (GstRTPBaseDepayload * depay, GstRTPBuffer * rtp) } GST_DEBUG_OBJECT (depay, "Found the start of the layer"); - if (self->stop_lost_events) { - send_last_lost_event_if_needed (self, picture_id); - self->stop_lost_events = FALSE; - } + if (!sent_lost_event) + send_lost_event_if_needed (self, picture_id, GST_BUFFER_PTS (rtp->buffer)); self->started = TRUE; + self->stop_lost_events = FALSE; + self->inter_picture = FALSE; } payload = gst_rtp_buffer_get_payload_subbuffer (rtp, hdrsize, -1); @@ -507,8 +492,7 @@ gst_rtp_vp9_depay_process (GstRTPBaseDepayload * depay, GstRTPBuffer * rtp) if (self->inter_picture) { GST_BUFFER_FLAG_SET (out, GST_BUFFER_FLAG_DELTA_UNIT); - if (self->waiting_for_keyframe) { - gst_rtp_base_depayload_flush (depay, FALSE); + if (!self->caps_sent) { gst_buffer_unref (out); out = NULL; GST_INFO_OBJECT (self, "Dropping inter-frame before intra-frame"); @@ -538,27 +522,27 @@ gst_rtp_vp9_depay_process (GstRTPBaseDepayload * depay, GstRTPBuffer * rtp) gst_pad_set_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (depay), srccaps); gst_caps_unref (srccaps); + self->caps_sent = TRUE; self->last_width = self->ss_width; self->last_height = self->ss_height; self->ss_width = 0; self->ss_height = 0; } - - self->waiting_for_keyframe = FALSE; } if (picture_id != PICTURE_ID_NONE) self->stop_lost_events = TRUE; + + self->last_pushed_was_lost_event = FALSE; + return out; } done: - gst_rtp_base_depayload_dropped (depay); return NULL; too_small: GST_LOG_OBJECT (self, "Invalid rtp packet (too small), ignoring"); - gst_rtp_base_depayload_flush (depay, FALSE); gst_adapter_clear (self->adapter); self->started = FALSE; goto done; @@ -573,10 +557,13 @@ gst_rtp_vp9_depay_change_state (GstElement * element, GstStateChange transition) case GST_STATE_CHANGE_READY_TO_PAUSED: self->last_width = -1; self->last_height = -1; + self->caps_sent = FALSE; self->last_picture_id = PICTURE_ID_NONE; - gst_event_replace (&self->last_lost_event, NULL); + if (self->last_lost_event) { + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; + } self->stop_lost_events = FALSE; - self->waiting_for_keyframe = TRUE; break; default: break; @@ -597,7 +584,10 @@ gst_rtp_vp9_depay_handle_event (GstRTPBaseDepayload * depay, GstEvent * event) self->last_width = -1; self->last_height = -1; self->last_picture_id = PICTURE_ID_NONE; - gst_event_replace (&self->last_lost_event, NULL); + if (self->last_lost_event) { + gst_event_unref (self->last_lost_event); + self->last_lost_event = NULL; + } self->stop_lost_events = FALSE; break; default: @@ -613,28 +603,17 @@ static gboolean gst_rtp_vp9_depay_packet_lost (GstRTPBaseDepayload * depay, GstEvent * event) { GstRtpVP9Depay *self = GST_RTP_VP9_DEPAY (depay); - const GstStructure *s; - gboolean might_have_been_fec; - - s = gst_event_get_structure (event); - if (self->stop_lost_events) { - if (gst_structure_get_boolean (s, "might-have-been-fec", - &might_have_been_fec) - && might_have_been_fec) { - GST_DEBUG_OBJECT (depay, "Stopping lost event %" GST_PTR_FORMAT, event); - gst_event_replace (&self->last_lost_event, event); - return TRUE; - } - } else if (self->last_picture_id != PICTURE_ID_NONE) { - GstStructure *s = gst_event_writable_structure (self->last_lost_event); - - /* We are currently processing a picture, let's make sure the - * base depayloader doesn't drop this lost event */ - gst_structure_remove_field (s, "might-have-been-fec"); + GST_DEBUG_OBJECT (depay, "Stopping lost event %" GST_PTR_FORMAT, event); + if (self->last_lost_event) + gst_event_unref (self->last_lost_event); + self->last_lost_event = gst_event_ref (event); + return TRUE; } + self->last_pushed_was_lost_event = TRUE; + return GST_RTP_BASE_DEPAYLOAD_CLASS (gst_rtp_vp9_depay_parent_class)->packet_lost (depay, event); -} +} \ No newline at end of file diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.h b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.h index 8c98a5bf186..fd5bbceb3f3 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.h +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9depay.h @@ -40,6 +40,7 @@ G_BEGIN_DECLS #define GST_RTP_VP9_DEPAY_GET_CLASS(obj) \ (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_RTP_VP9_DEPAY, \ GstRtpVP9DepayClass)) +#define GST_RTP_VP9_DEPAY_CAST(obj) ((GstRtpVP9Depay *)(obj)) typedef struct _GstRtpVP9Depay GstRtpVP9Depay; typedef struct _GstRtpVP9DepayClass GstRtpVP9DepayClass; @@ -61,6 +62,8 @@ struct _GstRtpVP9Depay gint last_height; guint last_picture_id; GstEvent *last_lost_event; + gboolean last_pushed_was_lost_event; + gboolean caps_sent; /* In between pictures, we might store GstRTPPacketLost events instead * of forwarding them immediately, we check upon reception of a new * picture id whether a gap was introduced, in which case we do forward @@ -70,15 +73,12 @@ struct _GstRtpVP9Depay gboolean stop_lost_events; gboolean inter_picture; - gboolean waiting_for_keyframe; - - /* Properties */ - gboolean wait_for_keyframe; - gboolean request_keyframe; + /* properties */ + gboolean hide_picture_id_gap; }; GType gst_rtp_vp9_depay_get_type (void); G_END_DECLS -#endif /* #ifndef __GST_RTP_VP9_DEPAY_H__ */ +#endif /* #ifndef __GST_RTP_VP9_DEPAY_H__ */ \ No newline at end of file diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.c index f5db16b2cf2..8aeb88d114e 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.c @@ -42,6 +42,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_rtp_vp9_pay_debug); #define DEFAULT_PICTURE_ID_MODE VP9_PAY_NO_PICTURE_ID #define DEFAULT_PICTURE_ID_OFFSET (-1) +#define DEFAULT_PARSE_FRAMES TRUE enum { @@ -49,6 +50,7 @@ enum PROP_PICTURE_ID, PROP_PICTURE_ID_MODE, PROP_PICTURE_ID_OFFSET, + PROP_PARSE_FRAMES, }; #define GST_TYPE_RTP_VP9_PAY_PICTURE_ID_MODE (gst_rtp_vp9_pay_picture_id_mode_get_type()) @@ -136,6 +138,7 @@ gst_rtp_vp9_pay_init (GstRtpVP9Pay * obj) { obj->picture_id_mode = DEFAULT_PICTURE_ID_MODE; obj->picture_id_offset = DEFAULT_PICTURE_ID_OFFSET; + obj->parse_frames = DEFAULT_PARSE_FRAMES; gst_rtp_vp9_pay_picture_id_reset (obj); } @@ -181,6 +184,12 @@ gst_rtp_vp9_pay_class_init (GstRtpVP9PayClass * gst_rtp_vp9_pay_class) -1, 0x7FFF, DEFAULT_PICTURE_ID_OFFSET, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PARSE_FRAMES, + g_param_spec_boolean ("parse-frames", "Parse frames", + "Whether to parse/validate frame bitstreams", + DEFAULT_PARSE_FRAMES, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT_ONLY)); + gst_element_class_add_static_pad_template (element_class, &gst_rtp_vp9_pay_sink_template); gst_element_class_add_static_pad_template (element_class, @@ -215,6 +224,9 @@ gst_rtp_vp9_pay_set_property (GObject * object, rtpvp9pay->picture_id_offset = g_value_get_int (value); gst_rtp_vp9_pay_picture_id_reset (rtpvp9pay); break; + case PROP_PARSE_FRAMES: + rtpvp9pay->parse_frames = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -237,6 +249,9 @@ gst_rtp_vp9_pay_get_property (GObject * object, case PROP_PICTURE_ID_OFFSET: g_value_set_int (value, rtpvp9pay->picture_id_offset); break; + case PROP_PARSE_FRAMES: + g_value_set_boolean (value, rtpvp9pay->parse_frames); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -459,9 +474,12 @@ gst_rtp_vp9_create_header_buffer (GstRtpVP9Pay * self, hdrlen, 0, 0); gst_rtp_buffer_map (out, GST_MAP_READWRITE, &rtpbuffer); p = gst_rtp_buffer_get_payload (&rtpbuffer); + + /* Zero-initialise required first byte*/ p[0] = 0x0; if (self->picture_id_mode != VP9_PAY_NO_PICTURE_ID) { + /* I=1 */ p[0] |= 0x80; if (self->picture_id_mode == VP9_PAY_PICTURE_ID_7BITS) { /* M=0 */ @@ -474,13 +492,17 @@ gst_rtp_vp9_create_header_buffer (GstRtpVP9Pay * self, } if (!self->is_keyframe) + /* P=1 */ p[0] |= 0x40; if (start) + /* B=1 */ p[0] |= 0x08; if (mark) + /* E=1 */ p[0] |= 0x04; if (self->is_keyframe && start) { + /* V=1 */ p[0] |= 0x02; /* scalability structure, hard coded for now to be similar to chromium for * quick and dirty interop */ @@ -570,12 +592,17 @@ gst_rtp_vp9_pay_handle_buffer (GstRTPBasePayload * payload, GstBuffer * buffer) size = gst_buffer_get_size (buffer); delta_unit = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT); - if (G_UNLIKELY (!gst_rtp_vp9_pay_parse_frame (self, buffer, size))) { - GST_ELEMENT_ERROR (self, STREAM, ENCODE, (NULL), - ("Failed to parse VP9 frame")); - return GST_FLOW_ERROR; + if (self->parse_frames) { + if (G_UNLIKELY (!gst_rtp_vp9_pay_parse_frame (self, buffer, size))) { + GST_ELEMENT_ERROR (self, STREAM, ENCODE, (NULL), + ("Failed to parse VP9 frame")); + return GST_FLOW_ERROR; + } + } else { + self->is_keyframe = !GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT); } + mtu = GST_RTP_BASE_PAYLOAD_MTU (payload); vp9_hdr_len = gst_rtp_vp9_calc_header_len (self, TRUE); max_paylen = gst_rtp_buffer_calc_payload_len (mtu - vp9_hdr_len, 0, 0); @@ -607,6 +634,30 @@ gst_rtp_vp9_pay_handle_buffer (GstRTPBasePayload * payload, GstBuffer * buffer) return ret; } +static void +gst_rtp_vp9_pay_set_resolution_from_event (GstRtpVP9Pay * self, + const GstEvent * event) +{ + GstCaps * caps = NULL; + GstStructure * s = NULL; + + gst_event_parse_caps (event, &caps); + s = gst_caps_get_structure (caps, 0); + + gint width = 0, height = 0; + if (gst_structure_get_int (s, "width", &width) + && gst_structure_get_int (s, "height", &height) + && width > 0 && height > 0) { + GST_INFO_OBJECT (self, "Parsed resolution from caps: %dx%d -> %dx%d", + self->width, self->height, width, height); + self->width = width; + self->height = height; + } else { + GST_INFO_OBJECT (self, "Failed to parse resolution from caps: " + "%dx%d -/-> %dx%d",self->width, self->height, width, height); + } +} + static gboolean gst_rtp_vp9_pay_sink_event (GstRTPBasePayload * payload, GstEvent * event) { @@ -614,10 +665,12 @@ gst_rtp_vp9_pay_sink_event (GstRTPBasePayload * payload, GstEvent * event) GstEventType event_type = GST_EVENT_TYPE (event); if (event_type == GST_EVENT_GAP || event_type == GST_EVENT_FLUSH_START) { - gint picture_id = self->picture_id; - gst_rtp_vp9_pay_picture_id_increment (self); - GST_DEBUG_OBJECT (payload, "Incrementing picture ID on %s event %d -> %d", - GST_EVENT_TYPE_NAME (event), picture_id, self->picture_id); + gint picture_id = self->picture_id; + gst_rtp_vp9_pay_picture_id_increment (self); + GST_DEBUG_OBJECT (payload, "Incrementing picture ID on %s event %d -> %d", + GST_EVENT_TYPE_NAME (event), picture_id, self->picture_id); + } else if (event_type == GST_EVENT_CAPS && !self->parse_frames) { + gst_rtp_vp9_pay_set_resolution_from_event (self, event); } return GST_RTP_BASE_PAYLOAD_CLASS (gst_rtp_vp9_pay_parent_class)->sink_event diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.h b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.h index b2ad88445f9..eef054df58a 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.h +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtpvp9pay.h @@ -62,6 +62,7 @@ struct _GstRtpVP9Pay GstVP9RtpPayPictureIDMode picture_id_mode; gint picture_id_offset; gint picture_id; + gboolean parse_frames; }; GType gst_rtp_vp9_pay_get_type (void); diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c index d5adf0f64a3..d0bd4ef6589 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c @@ -288,6 +288,7 @@ enum SIGNAL_REQUEST_PT_MAP, SIGNAL_PAYLOAD_TYPE_CHANGE, SIGNAL_CLEAR_PT_MAP, + SIGNAL_CLEAR_PT_MAP_FOR_SESSION_ID, SIGNAL_RESET_SYNC, SIGNAL_GET_SESSION, SIGNAL_GET_INTERNAL_SESSION, @@ -1059,32 +1060,40 @@ gst_rtp_bin_reset_sync (GstRtpBin * rtpbin) GST_RTP_BIN_UNLOCK (rtpbin); } +static void +gst_rtp_bin_clear_pt_map_for_session (GstRtpBin * bin, + GstRtpBinSession * session) +{ + GSList *streams; + + GST_DEBUG_OBJECT (bin, "clearing session %p", session); + g_signal_emit_by_name (session->session, "clear-pt-map", NULL); + + GST_RTP_SESSION_LOCK (session); + g_hash_table_foreach_remove (session->ptmap, return_true, NULL); + + for (streams = session->streams; streams; streams = g_slist_next (streams)) { + GstRtpBinStream *stream = (GstRtpBinStream *) streams->data; + + GST_DEBUG_OBJECT (bin, "clearing stream %p", stream); + if (g_signal_lookup ("clear-pt-map", G_OBJECT_TYPE (stream->buffer)) != 0) + g_signal_emit_by_name (stream->buffer, "clear-pt-map", NULL); + if (stream->demux) + g_signal_emit_by_name (stream->demux, "clear-pt-map", NULL); + } + GST_RTP_SESSION_UNLOCK (session); +} + static void gst_rtp_bin_clear_pt_map (GstRtpBin * bin) { - GSList *sessions, *streams; + GSList *sessions; GST_RTP_BIN_LOCK (bin); GST_DEBUG_OBJECT (bin, "clearing pt map"); for (sessions = bin->sessions; sessions; sessions = g_slist_next (sessions)) { GstRtpBinSession *session = (GstRtpBinSession *) sessions->data; - - GST_DEBUG_OBJECT (bin, "clearing session %p", session); - g_signal_emit_by_name (session->session, "clear-pt-map", NULL); - - GST_RTP_SESSION_LOCK (session); - g_hash_table_foreach_remove (session->ptmap, return_true, NULL); - - for (streams = session->streams; streams; streams = g_slist_next (streams)) { - GstRtpBinStream *stream = (GstRtpBinStream *) streams->data; - - GST_DEBUG_OBJECT (bin, "clearing stream %p", stream); - if (g_signal_lookup ("clear-pt-map", G_OBJECT_TYPE (stream->buffer)) != 0) - g_signal_emit_by_name (stream->buffer, "clear-pt-map", NULL); - if (stream->demux) - g_signal_emit_by_name (stream->demux, "clear-pt-map", NULL); - } - GST_RTP_SESSION_UNLOCK (session); + gst_rtp_bin_clear_pt_map_for_session (bin, session); } GST_RTP_BIN_UNLOCK (bin); @@ -1092,6 +1101,20 @@ gst_rtp_bin_clear_pt_map (GstRtpBin * bin) gst_rtp_bin_reset_sync (bin); } +static void +gst_rtp_bin_clear_pt_map_for_session_id (GstRtpBin * bin, guint session_id) +{ + GstRtpBinSession *session; + + GST_RTP_BIN_LOCK (bin); + GST_DEBUG_OBJECT (bin, "retrieving GstRtpSession, index: %u", session_id); + session = find_session_by_id (bin, (gint) session_id); + if (session) { + gst_rtp_bin_clear_pt_map_for_session (bin, session); + } + GST_RTP_BIN_UNLOCK (bin); +} + static GstElement * gst_rtp_bin_get_session (GstRtpBin * bin, guint session_id) { @@ -1511,9 +1534,9 @@ gst_rtp_bin_associate (GstRtpBin * bin, GstRtpBinStream * stream, guint8 len, running_time_rtp = last_extrtptime - base_rtptime; GST_DEBUG_OBJECT (bin, - "base %" G_GUINT64_FORMAT ", extrtptime %" G_GUINT64_FORMAT - ", local RTP %" G_GUINT64_FORMAT ", clock-rate %d, " - "clock-base %" G_GINT64_FORMAT, base_rtptime, + "base_time %" G_GUINT64_FORMAT ", base_rtptime %" G_GUINT64_FORMAT + ", extrtptime %" G_GUINT64_FORMAT ", local RTP %" G_GUINT64_FORMAT + ", clock-rate %d, clock-base %" G_GINT64_FORMAT, base_time, base_rtptime, last_extrtptime, running_time_rtp, clock_rate, rtp_clock_base); /* calculate local RTP time in gstreamer timestamp, we essentially perform the @@ -1524,7 +1547,10 @@ gst_rtp_bin_associate (GstRtpBin * bin, GstRtpBinStream * stream, guint8 len, gst_util_uint64_scale_int (running_time_rtp, GST_SECOND, clock_rate); running_time += base_time; - stream->have_sync = TRUE; + /* A sender that has no notion of wallclock or elapsed time MAY set the NTP + timestamp to zero. It makes sense to not synchronize this stream.*/ + if (ntpnstime != 0) + stream->have_sync = TRUE; GST_DEBUG_OBJECT (bin, "SR RTP running time %" G_GUINT64_FORMAT ", SR NTP %" G_GUINT64_FORMAT, @@ -2231,6 +2257,20 @@ gst_rtp_bin_class_init (GstRtpBinClass * klass) G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, G_STRUCT_OFFSET (GstRtpBinClass, clear_pt_map), NULL, NULL, NULL, G_TYPE_NONE, 0, G_TYPE_NONE); + /** + * GstRtpBin::clear-pt-map-for-session-id: + * @rtpbin: the object which received the signal + * @id: the session id + * + * Clear all previously cached pt-mapping obtained with + * #GstRtpBin::request-pt-map for session @id. + */ + gst_rtp_bin_signals[SIGNAL_CLEAR_PT_MAP_FOR_SESSION_ID] = + g_signal_new ("clear-pt-map-for-session-id", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, G_STRUCT_OFFSET (GstRtpBinClass, + clear_pt_map_for_session_id), NULL, NULL, + g_cclosure_marshal_generic, G_TYPE_NONE, 1, G_TYPE_UINT); + /** * GstRtpBin::reset-sync: * @rtpbin: the object which received the signal @@ -3056,6 +3096,8 @@ gst_rtp_bin_class_init (GstRtpBinClass * klass) gstbin_class->handle_message = GST_DEBUG_FUNCPTR (gst_rtp_bin_handle_message); klass->clear_pt_map = GST_DEBUG_FUNCPTR (gst_rtp_bin_clear_pt_map); + klass->clear_pt_map_for_session_id = + GST_DEBUG_FUNCPTR (gst_rtp_bin_clear_pt_map_for_session_id); klass->reset_sync = GST_DEBUG_FUNCPTR (gst_rtp_bin_reset_sync); klass->get_session = GST_DEBUG_FUNCPTR (gst_rtp_bin_get_session); klass->get_internal_session = diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.h b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.h index bc7c7e5f66d..c2d0b653236 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.h @@ -120,6 +120,7 @@ struct _GstRtpBinClass { /* action signals */ void (*clear_pt_map) (GstRtpBin *rtpbin); + void (*clear_pt_map_for_session_id) (GstRtpBin *rtpbin, guint session); void (*reset_sync) (GstRtpBin *rtpbin); GstElement* (*get_session) (GstRtpBin *rtpbin, guint session); RTPSession* (*get_internal_session) (GstRtpBin *rtpbin, guint session); diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c index 841b55b00fb..3bf9d19d696 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpfunnel.c @@ -53,12 +53,6 @@ * RTP funnel can than make sure that this event hits the right encoder based * on the SSRC embedded in the event. * - * Another feature of the RTP funnel is that it will mux together TWCC - * (Transport-Wide Congestion Control) sequence-numbers. The point being that - * it should increment "transport-wide", meaning potentially several - * bundled streams. Note that not *all* streams being bundled needs to be - * affected by this. As an example Google WebRTC will use bundle with audio - * and video, but will only use TWCC sequence-numbers for the video-stream(s). * */ @@ -86,7 +80,9 @@ struct _GstRtpFunnelPad { GstPad pad; guint32 ssrc; - gboolean has_twcc; + GstRTPBufferFlags buffer_flag; + GstClockTime us_latency; + gboolean has_latency; }; G_DEFINE_TYPE (GstRtpFunnelPad, gst_rtp_funnel_pad, GST_TYPE_PAD); @@ -103,6 +99,55 @@ gst_rtp_funnel_pad_init (G_GNUC_UNUSED GstRtpFunnelPad * pad) { } +static void +gst_rtp_funnel_pad_set_buffer_flag (GstRtpFunnelPad * pad, GstBuffer * buf) +{ + if (pad->buffer_flag) + GST_BUFFER_FLAG_SET (buf, pad->buffer_flag); +} + +static void +gst_rtp_funnel_pad_set_media_type (GstRtpFunnelPad * pad, + const GstStructure * s) +{ + const gchar *media_type = gst_structure_get_string (s, "media"); + if (g_strcmp0 (media_type, "audio") == 0) + pad->buffer_flag = GST_RTP_BUFFER_FLAG_MEDIA_AUDIO; + else if (g_strcmp0 (media_type, "video") == 0) + pad->buffer_flag = GST_RTP_BUFFER_FLAG_MEDIA_VIDEO; +} + +static gboolean +gst_rtp_funnel_pad_query_latency (GstRtpFunnelPad * pad, + gboolean * _live, GstClockTime * _max) +{ + GstQuery *query = gst_query_new_latency (); + gboolean res; + + /* Ask peer for latency */ + res = gst_pad_peer_query (GST_PAD_CAST (pad), query); + + if (res) { + gboolean live; + GstClockTime min, max; + gst_query_parse_latency (query, &live, &min, &max); + + pad->us_latency = min; + GST_INFO_OBJECT (pad, "us_latency: %" GST_TIME_FORMAT, + GST_TIME_ARGS (pad->us_latency)); + + if (_live) + *_live = live; + if (_max) + *_max = max; + } + + pad->has_latency = TRUE; + + gst_query_unref (query); + return res; +} + /**************** GstRTPFunnel ****************/ enum @@ -129,9 +174,6 @@ struct _GstRtpFunnel /* The last pad data was chained on */ GstPad *current_pad; - guint twcc_pads; /* numer of sinkpads with negotiated twcc */ - GstRTPHeaderExtension *twcc_ext; - /* properties */ gint common_ts_offset; }; @@ -218,57 +260,21 @@ gst_rtp_funnel_forward_segment (GstRtpFunnel * funnel, GstPad * pad) return; } -static void -gst_rtp_funnel_set_twcc_seqnum (GstRtpFunnel * funnel, - GstPad * pad, GstBuffer ** buf) -{ - GstRtpFunnelPad *fpad = GST_RTP_FUNNEL_PAD_CAST (pad); - guint8 twcc_seq[2] = { 0, }; - GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; - guint ext_id = gst_rtp_header_extension_get_id (funnel->twcc_ext); - guint8 *existing; - guint size; - - if (!funnel->twcc_ext || !fpad->has_twcc) - return; - - *buf = gst_buffer_make_writable (*buf); - - gst_rtp_header_extension_write (funnel->twcc_ext, *buf, - GST_RTP_HEADER_EXTENSION_ONE_BYTE, *buf, twcc_seq, sizeof (twcc_seq)); - - if (!gst_rtp_buffer_map (*buf, GST_MAP_READWRITE, &rtp)) - goto map_failed; - - if (gst_rtp_buffer_get_extension_onebyte_header (&rtp, ext_id, - 0, (gpointer) & existing, &size)) { - if (size >= gst_rtp_header_extension_get_max_size (funnel->twcc_ext, *buf)) { - existing[0] = twcc_seq[0]; - existing[1] = twcc_seq[1]; - } - } - /* TODO: two-byte variant */ - - gst_rtp_buffer_unmap (&rtp); - - return; - -map_failed: - { - GST_ERROR ("failed to map buffer %p", *buf); - } -} - static GstFlowReturn gst_rtp_funnel_sink_chain_object (GstPad * pad, GstRtpFunnel * funnel, gboolean is_list, GstMiniObject * obj) { + GstRtpFunnelPad *fpad = GST_RTP_FUNNEL_PAD_CAST (pad); GstFlowReturn res; GST_DEBUG_OBJECT (pad, "received %" GST_PTR_FORMAT, obj); GST_PAD_STREAM_LOCK (funnel->srcpad); + if (!fpad->has_latency) { + gst_rtp_funnel_pad_query_latency (fpad, NULL, NULL); + } + gst_rtp_funnel_send_sticky (funnel, pad); gst_rtp_funnel_forward_segment (funnel, pad); @@ -276,7 +282,8 @@ gst_rtp_funnel_sink_chain_object (GstPad * pad, GstRtpFunnel * funnel, res = gst_pad_push_list (funnel->srcpad, GST_BUFFER_LIST_CAST (obj)); } else { GstBuffer *buf = GST_BUFFER_CAST (obj); - gst_rtp_funnel_set_twcc_seqnum (funnel, pad, &buf); + gst_rtp_funnel_pad_set_buffer_flag (fpad, buf); + GST_BUFFER_PTS (buf) += fpad->us_latency; res = gst_pad_push (funnel->srcpad, buf); } GST_PAD_STREAM_UNLOCK (funnel->srcpad); @@ -303,33 +310,61 @@ gst_rtp_funnel_sink_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) GST_MINI_OBJECT_CAST (buffer)); } -static void -gst_rtp_funnel_set_twcc_ext_id (GstRtpFunnel * funnel, guint8 twcc_ext_id) +static gboolean +gst_rtp_funnel_query_latency (GstRtpFunnel * funnel, GstQuery * query) { - gchar *name; - guint current_ext_id; - - current_ext_id = gst_rtp_header_extension_get_id (funnel->twcc_ext); - g_object_set (funnel->twcc_ext, "n-streams", funnel->twcc_pads, NULL); - - if (current_ext_id == twcc_ext_id) - return; - - name = g_strdup_printf ("extmap-%u", twcc_ext_id); - - gst_caps_set_simple (funnel->srccaps, name, G_TYPE_STRING, - gst_rtp_header_extension_get_uri (funnel->twcc_ext), NULL); + GstClockTime max = GST_CLOCK_TIME_NONE; + gboolean live = FALSE, done = FALSE; + GValue item = G_VALUE_INIT; + GstIterator *it = gst_element_iterate_sink_pads (GST_ELEMENT_CAST (funnel)); + + /* Take maximum of all latency values */ + while (!done) { + GstIteratorResult ires = gst_iterator_next (it, &item); + switch (ires) { + case GST_ITERATOR_DONE: + done = TRUE; + break; + case GST_ITERATOR_OK: + { + GstRtpFunnelPad *pad = g_value_get_object (&item); + gboolean live_cur; + GstClockTime max_cur; + + if (gst_rtp_funnel_pad_query_latency (pad, &live_cur, &max_cur)) { + /* max is the buffering-potential upstream, and we are only + ever as good as our weakest link here, so take the smallest + of the max values */ + max = MIN (max_cur, max); + /* if one branch is live, we are live */ + live = live || live_cur; + } + g_value_reset (&item); + break; + } + case GST_ITERATOR_RESYNC: + max = GST_CLOCK_TIME_NONE; + live = FALSE; + gst_iterator_resync (it); + break; + default: + done = TRUE; + break; + } + } + g_value_unset (&item); + gst_iterator_free (it); - g_free (name); + /* since we terminate upstream latency, we report 0 downstream */ + gst_query_set_latency (query, live, 0, max); - /* make sure we update the sticky with the new caps */ - funnel->send_sticky_events = TRUE; + /* store the results */ + GST_INFO_OBJECT (funnel, "Replying to latency query with: %" GST_PTR_FORMAT, + query); - gst_rtp_header_extension_set_id (funnel->twcc_ext, twcc_ext_id); + return TRUE; } -#define TWCC_EXTMAP_STR "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01" - static gboolean gst_rtp_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) { @@ -351,20 +386,16 @@ gst_rtp_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) GstCaps *caps; GstStructure *s; guint ssrc; - guint8 ext_id; - GstCaps *rtpcaps = gst_caps_new_empty_simple (RTP_CAPS); gst_event_parse_caps (event, &caps); GST_OBJECT_LOCK (funnel); - if (!gst_caps_can_intersect (rtpcaps, caps)) { + if (!gst_caps_can_intersect (funnel->srccaps, caps)) { GST_ERROR_OBJECT (funnel, "Can't intersect with caps %" GST_PTR_FORMAT, caps); g_assert_not_reached (); } - gst_caps_unref (rtpcaps); - s = gst_caps_get_structure (caps, 0); if (gst_structure_get_uint (s, "ssrc", &ssrc)) { fpad->ssrc = ssrc; @@ -372,21 +403,17 @@ gst_rtp_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) g_hash_table_insert (funnel->ssrc_to_pad, GUINT_TO_POINTER (ssrc), pad); } - if (!funnel->twcc_ext) - funnel->twcc_ext = - gst_rtp_header_extension_create_from_uri (TWCC_EXTMAP_STR); - - ext_id = gst_rtp_get_extmap_id_for_attribute (s, TWCC_EXTMAP_STR); - if (ext_id > 0) { - fpad->has_twcc = TRUE; - funnel->twcc_pads++; - gst_rtp_funnel_set_twcc_ext_id (funnel, ext_id); - } + gst_rtp_funnel_pad_set_media_type (fpad, s); GST_OBJECT_UNLOCK (funnel); forward = FALSE; break; } + case GST_EVENT_LATENCY_CHANGED: + { + gst_rtp_funnel_pad_query_latency (fpad, NULL, NULL); + break; + } default: break; } @@ -411,17 +438,15 @@ gst_rtp_funnel_sink_query (GstPad * pad, GstObject * parent, GstQuery * query) { GstCaps *filter_caps; GstCaps *new_caps; - GstCaps *rtpcaps = gst_caps_new_empty_simple (RTP_CAPS); gst_query_parse_caps (query, &filter_caps); GST_OBJECT_LOCK (funnel); if (filter_caps) { - new_caps = gst_caps_intersect_full (rtpcaps, filter_caps, + new_caps = gst_caps_intersect_full (funnel->srccaps, filter_caps, GST_CAPS_INTERSECT_FIRST); - gst_caps_unref (rtpcaps); } else { - new_caps = rtpcaps; + new_caps = gst_caps_copy (funnel->srccaps); } GST_OBJECT_UNLOCK (funnel); @@ -502,6 +527,23 @@ gst_rtp_funnel_src_event (GstPad * pad, GstObject * parent, GstEvent * event) return ret; } +static gboolean +gst_rtp_funnel_src_query (GstPad * pad, GstObject * parent, GstQuery * query) +{ + GstRtpFunnel *funnel = GST_RTP_FUNNEL_CAST (parent); + gboolean res = FALSE; + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_LATENCY: + res = gst_rtp_funnel_query_latency (funnel, query); + break; + default: + res = gst_pad_query_default (pad, parent, query); + break; + } + return res; +} + static GstPad * gst_rtp_funnel_request_new_pad (GstElement * element, GstPadTemplate * templ, const gchar * name, const GstCaps * caps) @@ -617,9 +659,6 @@ gst_rtp_funnel_finalize (GObject * object) gst_caps_unref (funnel->srccaps); g_hash_table_destroy (funnel->ssrc_to_pad); - - gst_clear_object (&funnel->twcc_ext); - G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -664,6 +703,9 @@ gst_rtp_funnel_init (GstRtpFunnel * funnel) gst_pad_use_fixed_caps (funnel->srcpad); gst_pad_set_event_function (funnel->srcpad, GST_DEBUG_FUNCPTR (gst_rtp_funnel_src_event)); + gst_pad_set_query_function (funnel->srcpad, + GST_DEBUG_FUNCPTR (gst_rtp_funnel_src_query)); + gst_element_add_pad (GST_ELEMENT (funnel), funnel->srcpad); funnel->send_sticky_events = TRUE; diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.c new file mode 100644 index 00000000000..29cc4fc2656 --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.c @@ -0,0 +1,390 @@ +/* GStreamer + * Copyright (C) <2021> Havard Graff + * Camilo Celis Guzman + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more + */ + +/** + * SECTION:element-rtphdrextroi + * @title: rtphdrextroi + * @short_description: Region of interest (ROI) RTP Header Extension + * + * + * Since: 1.20 + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstrtphdrext-roi.h" +#include + +GST_DEBUG_CATEGORY_STATIC (rtp_hdrext_roi_debug); +#define GST_CAT_DEFAULT rtp_hdrext_roi_debug + +/* + * Internal roi-ext-hdr ID used as a safety mechanism, to make sure + * that we not only only care about a specific roi_type (as in + * GstVideoRegionOfInterestMeta) but also that it has been payloaded + * and payloaded using the set of callbacks implemented below. + */ +#define ROI_HDR_EXT_URI GST_RTP_HDREXT_BASE"TBD:draft-ford-avtcore-roi-extension-00" + +#define ROI_EXTHDR_SIZE 11 +#define ROI_MAX_ROI_TYPES 15 /* 0 is unused and we allocate 1 byte for the ID */ + +#define DEFAULT_ROI_TYPE 126 +#define DEFAULT_ROI_ID 8 + +enum +{ + PROP_0, + PROP_ROI_TYPES, +}; + +struct _GstRTPHeaderExtensionRoi +{ + GstRTPHeaderExtension parent; + + GHashTable *roi_types; + GHashTable *roi_ids; +}; + +G_DEFINE_TYPE_WITH_CODE (GstRTPHeaderExtensionRoi, + gst_rtp_header_extension_roi, GST_TYPE_RTP_HEADER_EXTENSION, + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "rtphdrextroi", 0, + "RTP ROI Header Extensions")); +GST_ELEMENT_REGISTER_DEFINE (rtphdrextroi, "rtphdrextroi", + GST_RANK_MARGINAL, GST_TYPE_RTP_HEADER_EXTENSION_ROI); + +static void +gst_rtp_header_extension_roi_serialize_roi_types (GstRTPHeaderExtensionRoi * + self, GValue * value) +{ + GHashTableIter iter; + GValue val = G_VALUE_INIT; + gpointer key; + + GST_DEBUG_OBJECT (self, "Serialize roi-types %p", self->roi_types); + + g_value_init (&val, G_TYPE_UINT); + g_hash_table_iter_init (&iter, self->roi_types); + + while (g_hash_table_iter_next (&iter, &key, NULL)) { + guint roi_type = GPOINTER_TO_UINT (key); + g_value_set_uint (&val, roi_type); + gst_value_array_append_value (value, &val); + } + + g_value_unset (&val); +} + +static void +gst_rtp_header_extension_roi_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (object); + switch (prop_id) { + case PROP_ROI_TYPES: + gst_rtp_header_extension_roi_serialize_roi_types (self, value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_warn_invalid_roi_type (const GValue * val) +{ + GValue str = G_VALUE_INIT; + + g_value_init (&str, G_TYPE_STRING); + g_value_transform (val, &str); + + g_warning ("Invalid roi-type must be an unsigned int got '%s'", + g_value_get_string (&str)); + + g_value_unset (&str); +} + +static void +gst_rtp_header_extension_roi_reset_roi_types (GstRTPHeaderExtensionRoi * self) +{ + GST_DEBUG_OBJECT (self, "Re-set roi-types"); + + if (self->roi_types) + g_hash_table_destroy (self->roi_types); + if (self->roi_ids) + g_hash_table_destroy (self->roi_ids); + self->roi_types = g_hash_table_new (NULL, NULL); + self->roi_ids = g_hash_table_new (NULL, NULL); +} + +static void +gst_rtp_header_extension_roi_set_roi_types (GstRTPHeaderExtensionRoi * self, + const GValue * value) +{ + guint num_roi_types = gst_value_array_get_size (value); + + GST_DEBUG_OBJECT (self, "Set roi-types with %p", value); + + if (num_roi_types == 0) { + GST_INFO_OBJECT (self, "roi-types length is 0. Only the default roi-type " + "of %u will be allowed", DEFAULT_ROI_TYPE); + gst_rtp_header_extension_roi_reset_roi_types (self); + g_hash_table_insert (self->roi_types, + GUINT_TO_POINTER (DEFAULT_ROI_TYPE), GUINT_TO_POINTER (DEFAULT_ROI_ID)); + g_hash_table_insert (self->roi_ids, + GUINT_TO_POINTER (DEFAULT_ROI_ID), GUINT_TO_POINTER (DEFAULT_ROI_TYPE)); + return; + } + + /* verify our maximum number of roi-types supported: + * we can only payload 1 RoI meta per roi-type; and given the a fixed size + * per header extension we allowed a finite set of roi-tyes */ + if (num_roi_types > ROI_MAX_ROI_TYPES) { + g_warning ("Maximum allowed set of roi-types surpassed. " + "Current length=%d max-length=%d", num_roi_types, ROI_MAX_ROI_TYPES); + return; + } + + /* convert given array into 2 hash talbes: a bi-directional mapping of + * roi-type to roi-id and vice versa. */ + gst_rtp_header_extension_roi_reset_roi_types (self); + for (guint i = 0; i < num_roi_types; i++) { + const GValue *val = gst_value_array_get_value (value, i); + guint roi_type = DEFAULT_ROI_TYPE; + guint roi_id = i + 1; /* avoid using 0 as roi-id */ + + if (!G_VALUE_HOLDS_UINT (val)) { + _warn_invalid_roi_type (val); + g_hash_table_destroy (self->roi_types); + g_hash_table_destroy (self->roi_ids); + return; + } + + roi_type = g_value_get_uint (val); + + /* we dont allow duplicates */ + if (g_hash_table_lookup (self->roi_types, GUINT_TO_POINTER (roi_type))) { + g_warning ("Invalid roi-types. No duplicates are allowed"); + g_hash_table_unref (self->roi_types); + g_hash_table_unref (self->roi_ids); + } + + GST_TRACE_OBJECT (self, "Adding %u:%u as roi-type", roi_type, roi_id); + g_hash_table_insert (self->roi_types, + GUINT_TO_POINTER (roi_type), GUINT_TO_POINTER (roi_id)); + g_hash_table_insert (self->roi_ids, + GUINT_TO_POINTER (roi_id), GUINT_TO_POINTER (roi_type)); + } +} + +static void +gst_rtp_header_extension_roi_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (object); + switch (prop_id) { + case PROP_ROI_TYPES: + gst_rtp_header_extension_roi_set_roi_types (self, value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static GstRTPHeaderExtensionFlags +gst_rtp_header_extension_roi_get_supported_flags (GstRTPHeaderExtension * ext) +{ + // GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (ext); + // return g_hash_table_size (self->roi_types) >= 1 ? + // GST_RTP_HEADER_EXTENSION_ONE_BYTE : GST_RTP_HEADER_EXTENSION_TWO_BYTE; + return GST_RTP_HEADER_EXTENSION_TWO_BYTE; +} + +static gsize +gst_rtp_header_extension_roi_get_max_size (GstRTPHeaderExtension * ext, + const GstBuffer * input_meta) +{ + GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (ext); + return g_hash_table_size (self->roi_types) * ROI_EXTHDR_SIZE; +} + +static gssize +gst_rtp_header_extension_roi_write (GstRTPHeaderExtension * ext, + const GstBuffer * input_meta, GstRTPHeaderExtensionFlags write_flags, + G_GNUC_UNUSED GstBuffer * output, guint8 * data, gsize size) +{ + GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (ext); + GstVideoRegionOfInterestMeta *meta; + gpointer state = NULL; + guint offset = 0; + + g_return_val_if_fail (size >= + gst_rtp_header_extension_roi_get_max_size (ext, NULL), -1); + g_return_val_if_fail (write_flags & + gst_rtp_header_extension_roi_get_supported_flags (ext), -1); + + gboolean written_roi_ids[ROI_MAX_ROI_TYPES + 1] = { 0 }; + while ((meta = (GstVideoRegionOfInterestMeta *) + gst_buffer_iterate_meta_filtered ((GstBuffer *) input_meta, &state, + GST_VIDEO_REGION_OF_INTEREST_META_API_TYPE))) { + GstStructure *extra_param_s; + guint num_faces = 0; + + /* filter by known roi-types */ + gpointer roi_id_ptr = g_hash_table_lookup (self->roi_types, + GUINT_TO_POINTER (meta->roi_type)); + guint roi_id = GPOINTER_TO_UINT (roi_id_ptr); + GST_TRACE_OBJECT (self, "Got roi-type %u roi-id %u", meta->roi_type, + roi_id); + + if (!roi_id_ptr) + continue; + + if (written_roi_ids[roi_id]) { + GST_TRACE_OBJECT (self, "Already written RoI meta with type %d. " + "Skipping.", meta->roi_type); + continue; + } + + /* RoI coordinates */ + GST_WRITE_UINT16_BE (&data[offset + 0], meta->x); + GST_WRITE_UINT16_BE (&data[offset + 2], meta->y); + GST_WRITE_UINT16_BE (&data[offset + 4], meta->w); + GST_WRITE_UINT16_BE (&data[offset + 6], meta->h); + + /* RoI type */ + g_assert (roi_id <= G_MAXUINT8); + GST_WRITE_UINT8 (&data[offset + 8], (guint8) roi_id); + + /* RoI extra: number of faces */ + extra_param_s = + gst_video_region_of_interest_meta_get_param (meta, "extra-param"); + if (extra_param_s) + gst_structure_get_uint (extra_param_s, "num_faces", &num_faces); + GST_WRITE_UINT16_BE (&data[offset + 9], num_faces); + + GST_TRACE_OBJECT (self, "Wrote header extension from RoI meta " + "(%u %u,%u %ux%u %u)", meta->roi_type, meta->x, meta->y, meta->w, + meta->h, num_faces); + + written_roi_ids[roi_id] = TRUE; + offset += ROI_EXTHDR_SIZE; + } + + return offset; +} + +static gboolean +gst_rtp_header_extension_roi_read (GstRTPHeaderExtension * ext, + GstRTPHeaderExtensionFlags read_flags, const guint8 * data, gsize size, + GstBuffer * buffer) +{ + GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (ext); + + /* read as many extension headers as we have signaled roi-types + * XXX: Is this a valid assumption? it makes this only work with + * GStreamer payloader */ + guint offset = 0; + for (guint i = 0; i < g_hash_table_size (self->roi_types); i++) { + guint16 roi_id = GST_READ_UINT8 (&data[offset + 8]); + /* filter by roi-id (roi-type stored in the header extension) */ + gpointer rtp_type_ptr = + g_hash_table_lookup (self->roi_ids, GUINT_TO_POINTER (roi_id)); + guint roi_type = GPOINTER_TO_UINT (rtp_type_ptr); + if (rtp_type_ptr) { + guint16 x = GST_READ_UINT16_BE (&data[offset + 0]); + guint16 y = GST_READ_UINT16_BE (&data[offset + 2]); + guint16 w = GST_READ_UINT16_BE (&data[offset + 4]); + guint16 h = GST_READ_UINT16_BE (&data[offset + 6]); + guint16 num_faces = GST_READ_UINT16_BE (&data[offset + 9]); + GstVideoRegionOfInterestMeta *meta = + gst_buffer_add_video_region_of_interest_meta_id (buffer, + GPOINTER_TO_UINT (rtp_type_ptr), x, y, w, h); + GstStructure *extra_param_s = gst_structure_new ("extra-param", + "num_faces", G_TYPE_UINT, num_faces, NULL); + gst_video_region_of_interest_meta_add_param (meta, extra_param_s); + + GST_TRACE_OBJECT (self, "Read header extension and added RoI meta " + "(%u %u,%u %ux%u %u) to buffer %p", roi_type, + x, y, w, h, num_faces, buffer); + } + offset += ROI_EXTHDR_SIZE; + } + return TRUE; +} + +static void +gst_rtp_header_extension_roi_dispose (GObject * object) +{ + GstRTPHeaderExtensionRoi *self = GST_RTP_HEADER_EXTENSION_ROI_CAST (object); + g_assert (self->roi_types); + g_assert (self->roi_ids); + g_hash_table_destroy (self->roi_types); + g_hash_table_destroy (self->roi_ids); + G_OBJECT_CLASS (gst_rtp_header_extension_roi_parent_class)->dispose (object); +} + +static void +gst_rtp_header_extension_roi_class_init (GstRTPHeaderExtensionRoiClass * klass) +{ + GstRTPHeaderExtensionClass *rtp_hdr_class; + GstElementClass *gstelement_class; + GObjectClass *gobject_class; + rtp_hdr_class = GST_RTP_HEADER_EXTENSION_CLASS (klass); + gobject_class = (GObjectClass *) klass; + gstelement_class = GST_ELEMENT_CLASS (klass); + gobject_class->dispose = + GST_DEBUG_FUNCPTR (gst_rtp_header_extension_roi_dispose); + gobject_class->get_property = gst_rtp_header_extension_roi_get_property; + gobject_class->set_property = gst_rtp_header_extension_roi_set_property; + /** + * rtphdrextroi:roi-type: + * + * What roi-type (GQuark) to write the extension-header for. + * + * Since: 1.20 + */ + g_object_class_install_property (gobject_class, PROP_ROI_TYPES, + gst_param_spec_array ("roi-types", "RoI types", + "Set of what roi-type(s) (GQuark(s)) to write the extension-header for", + g_param_spec_uint ("roi-type", "RoI type", + "What roi-type (GQuark) to write the extension-header for", + 1, G_MAXUINT32, DEFAULT_ROI_TYPE, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | + G_PARAM_STATIC_STRINGS), + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + rtp_hdr_class->get_supported_flags = + gst_rtp_header_extension_roi_get_supported_flags; + rtp_hdr_class->get_max_size = gst_rtp_header_extension_roi_get_max_size; + rtp_hdr_class->write = gst_rtp_header_extension_roi_write; + rtp_hdr_class->read = gst_rtp_header_extension_roi_read; + gst_element_class_set_static_metadata (gstelement_class, + "Region-of-Interest (ROI) RTP Header Extension", + GST_RTP_HDREXT_ELEMENT_CLASS, + "Region-of-Interest (ROI) RTP Header Extension", + "Havard Graff "); + gst_rtp_header_extension_class_set_uri (rtp_hdr_class, ROI_HDR_EXT_URI); + GST_DEBUG_CATEGORY_INIT (rtp_hdrext_roi_debug, "rtphdrextroi", 0, + "RTPHeaderExtensionRoI"); +} + +static void +gst_rtp_header_extension_roi_init (GstRTPHeaderExtensionRoi * self) +{ + GST_DEBUG_OBJECT (self, "creating element"); +} diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.h b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.h new file mode 100644 index 00000000000..ba7a93e8849 --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtphdrext-roi.h @@ -0,0 +1,34 @@ +/* GStreamer + * Copyright (C) <2021> Havard Graff + * Camilo Celis Guzman + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more + */ + +#ifndef __GST_RTPHDREXT_ROI_H__ +#define __GST_RTPHDREXT_ROI_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_RTP_HEADER_EXTENSION_ROI (gst_rtp_header_extension_roi_get_type()) + +G_DECLARE_FINAL_TYPE (GstRTPHeaderExtensionRoi, gst_rtp_header_extension_roi, GST, RTP_HEADER_EXTENSION_ROI, GstRTPHeaderExtension) + +GST_ELEMENT_REGISTER_DECLARE (rtphdrextroi); + +#define GST_RTP_HEADER_EXTENSION_ROI_CAST(obj) ((GstRTPHeaderExtensionRoi *)obj) + +G_END_DECLS + +#endif /* __GST_RTPHDREXT_ROI_H__ */ diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index 5c05386cc89..6bcd9a06759 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -105,6 +105,7 @@ #include #include #include +#include #include #include #include @@ -120,6 +121,8 @@ GST_DEBUG_CATEGORY (rtpjitterbuffer_debug); #define GST_CAT_DEFAULT (rtpjitterbuffer_debug) +#define NO_CLOCK_RATE_WARNING_THROTTLE 128 + /* RTPJitterBuffer signals and args */ enum { @@ -135,7 +138,7 @@ enum #define DEFAULT_DROP_ON_LATENCY FALSE #define DEFAULT_TS_OFFSET 0 #define DEFAULT_MAX_TS_OFFSET_ADJUSTMENT 0 -#define DEFAULT_DO_LOST FALSE +#define DEFAULT_DO_LOST TRUE #define DEFAULT_POST_DROP_MESSAGES FALSE #define DEFAULT_DROP_MESSAGES_INTERVAL_MS 200 #define DEFAULT_MODE RTP_JITTER_BUFFER_MODE_SLAVE @@ -286,6 +289,21 @@ enum #define GST_BUFFER_IS_RETRANSMISSION(buffer) \ GST_BUFFER_FLAG_IS_SET (buffer, GST_RTP_BUFFER_FLAG_RETRANSMISSION) +#define GST_BUFFER_IS_ULPFEC(buffer) \ + GST_BUFFER_FLAG_IS_SET (buffer, GST_RTP_BUFFER_FLAG_ULPFEC) + +#if !GLIB_CHECK_VERSION(2, 60, 0) +#define g_queue_clear_full queue_clear_full +static void +queue_clear_full (GQueue * queue, GDestroyNotify free_func) +{ + gpointer data; + + while ((data = g_queue_pop_head (queue)) != NULL) + free_func (data); +} +#endif + struct _GstRtpJitterBufferPrivate { GstPad *sinkpad, *srcpad; @@ -355,10 +373,14 @@ struct _GstRtpJitterBufferPrivate guint32 seqnum_base; /* last output time */ GstClockTime last_out_time; - /* last valid input timestamp and rtptime pair */ - GstClockTime ips_pts; guint64 ips_rtptime; + /* current packet spacing */ GstClockTime packet_spacing; + /* packet spacing calculated from the last received packet. + * Will update packet_spacing if we receive 2 consecutives packets with the + * same spacing. */ + GstClockTime prev_packet_spacing; + gint equidistant; GQueue gap_packets; @@ -390,6 +412,7 @@ struct _GstRtpJitterBufferPrivate gint32 clock_rate; gint64 clock_base; gint64 ts_offset_remainder; + guint no_clock_rate_count; /* when we are shutting down */ GstFlowReturn srcresult; @@ -428,8 +451,22 @@ struct _GstRtpJitterBufferPrivate GstClockTime last_pts; guint64 last_rtptime; GstClockTime last_ntpnstime; + guint64 jitter_count; + GstClockTime max_jitter; + GstClockTime min_jitter; + guint64 jitter_sum; + GstClockTime total_avg_jitter; GstClockTime avg_jitter; + /* for the latency estimation */ + guint estimated_latency_ms; + GstClockTimeDiff avg_clock_diff; + guint64 instant_max_jitter_top; + guint64 instant_max_jitter_bottom; + guint64 etimatied_max_jitter_top; + guint64 etimatied_max_jitter_bottom; + gfloat dencity_ratio; + /* for dropped packet messages */ GstClockTime last_drop_msg_timestamp; /* accumulators; reset every time a drop message is posted */ @@ -590,6 +627,22 @@ static void update_rtx_stats (GstRtpJitterBuffer * jitterbuffer, static GstClockTime get_current_running_time (GstRtpJitterBuffer * jitterbuffer); +static GQuark quark_application_x_rtp_jitterbuffer_stats; +static GQuark quark_num_pushed; +static GQuark quark_num_lost; +static GQuark quark_num_late; +static GQuark quark_num_duplicates; +static GQuark quark_avg_jitter; +static GQuark quark_total_avg_jitter; +static GQuark quark_min_jitter; +static GQuark quark_max_jitter; +static GQuark quark_latency_ms; +static GQuark quark_estimated_latency_ms; +static GQuark quark_rtx_count; +static GQuark quark_rtx_success_count; +static GQuark quark_rtx_per_packet; +static GQuark quark_rtx_rtt; + static void gst_rtp_jitter_buffer_class_init (GstRtpJitterBufferClass * klass) { @@ -599,6 +652,24 @@ gst_rtp_jitter_buffer_class_init (GstRtpJitterBufferClass * klass) gobject_class = (GObjectClass *) klass; gstelement_class = (GstElementClass *) klass; + quark_application_x_rtp_jitterbuffer_stats = + g_quark_from_static_string ("application/x-rtp-jitterbuffer-stats"); + quark_num_pushed = g_quark_from_static_string ("num-pushed"); + quark_num_lost = g_quark_from_static_string ("num-lost"); + quark_num_late = g_quark_from_static_string ("num-late"); + quark_num_duplicates = g_quark_from_static_string ("num-duplicates"); + quark_avg_jitter = g_quark_from_static_string ("avg-jitter"); + quark_total_avg_jitter = g_quark_from_static_string ("total-avg-jitter"); + quark_min_jitter = g_quark_from_static_string ("min-jitter"); + quark_max_jitter = g_quark_from_static_string ("max-jitter"); + quark_estimated_latency_ms = + g_quark_from_static_string ("estimated-latency-ms"); + quark_latency_ms = g_quark_from_static_string ("latency-ms"); + quark_rtx_count = g_quark_from_static_string ("rtx-count"); + quark_rtx_success_count = g_quark_from_static_string ("rtx-success-count"); + quark_rtx_per_packet = g_quark_from_static_string ("rtx-per-packet"); + quark_rtx_rtt = g_quark_from_static_string ("rtx-rtt"); + gobject_class->finalize = gst_rtp_jitter_buffer_finalize; gobject_class->set_property = gst_rtp_jitter_buffer_set_property; @@ -1129,6 +1200,7 @@ gst_rtp_jitter_buffer_init (GstRtpJitterBuffer * jitterbuffer) priv->add_reference_timestamp_meta = DEFAULT_ADD_REFERENCE_TIMESTAMP_META; priv->sync_interval = DEFAULT_SYNC_INTERVAL; + priv->no_clock_rate_count = 0; priv->ts_offset_remainder = 0; priv->last_dts = -1; priv->last_pts = -1; @@ -1137,6 +1209,13 @@ gst_rtp_jitter_buffer_init (GstRtpJitterBuffer * jitterbuffer) priv->last_known_ext_rtptime = -1; priv->last_known_ntpnstime = -1; priv->avg_jitter = 0; + priv->avg_clock_diff = -1; + priv->instant_max_jitter_top = 0; + priv->instant_max_jitter_bottom = 0; + priv->etimatied_max_jitter_top = 0; + priv->etimatied_max_jitter_bottom = 0; + priv->dencity_ratio = 0.5f; + priv->estimated_latency_ms = 0; priv->last_drop_msg_timestamp = GST_CLOCK_TIME_NONE; priv->num_too_late = 0; priv->num_drop_on_latency = 0; @@ -1506,12 +1585,11 @@ _get_cname_ssrc_mappings (GstRtpJitterBuffer * jitterbuffer, static gboolean gst_jitter_buffer_sink_parse_caps (GstRtpJitterBuffer * jitterbuffer, - GstCaps * caps, gint pt) + GstCaps * caps) { GstRtpJitterBufferPrivate *priv; GstStructure *caps_struct; guint val; - gint payload = -1; GstClockTime tval; const gchar *ts_refclk, *mediaclk; GstCaps *ts_meta_ref = NULL; @@ -1521,19 +1599,7 @@ gst_jitter_buffer_sink_parse_caps (GstRtpJitterBuffer * jitterbuffer, /* first parse the caps */ caps_struct = gst_caps_get_structure (caps, 0); - GST_DEBUG_OBJECT (jitterbuffer, "got caps %" GST_PTR_FORMAT, caps); - - if (gst_structure_get_int (caps_struct, "payload", &payload) && pt != -1 - && payload != pt) { - GST_ERROR_OBJECT (jitterbuffer, - "Got caps with wrong payload type (got %d, expected %d)", pt, payload); - return FALSE; - } - - if (payload != -1) { - GST_DEBUG_OBJECT (jitterbuffer, "Got payload type %d", payload); - priv->last_pt = payload; - } + GST_DEBUG_OBJECT (jitterbuffer, "got caps"); /* we need a clock-rate to convert the rtp timestamps to GStreamer time and to * measure the amount of data in the buffer */ @@ -1561,7 +1627,7 @@ gst_jitter_buffer_sink_parse_caps (GstRtpJitterBuffer * jitterbuffer, GST_DEBUG_OBJECT (jitterbuffer, "got clock-base %" G_GINT64_FORMAT, priv->clock_base); - if (gst_structure_get_uint (caps_struct, "seqnum-base", &val)) { + if (gst_structure_get_uint (caps_struct, "seqnum-offset", &val)) { /* first expected seqnum, only update when we didn't have a previous base. */ if (priv->next_in_seqnum == -1) priv->next_in_seqnum = val; @@ -1574,10 +1640,10 @@ gst_jitter_buffer_sink_parse_caps (GstRtpJitterBuffer * jitterbuffer, priv->seqnum_base = -1; } - GST_DEBUG_OBJECT (jitterbuffer, "got seqnum-base %d", priv->next_in_seqnum); + GST_DEBUG_OBJECT (jitterbuffer, "got seqnum-offset %d", priv->next_in_seqnum); - /* the start and stop times. The seqnum-base corresponds to the start time. We - * will keep track of the seqnums on the output and when we reach the one + /* the start and stop times. The seqnum-offset corresponds to the start time. + * We will keep track of the seqnums on the output and when we reach the one * corresponding to npt-stop, we emit the npt-stop-reached signal */ if (gst_structure_get_clock_time (caps_struct, "npt-start", &tval)) priv->npt_start = tval; @@ -1681,6 +1747,11 @@ gst_jitter_buffer_sink_parse_caps (GstRtpJitterBuffer * jitterbuffer, gst_rtp_get_extmap_id_for_attribute (caps_struct, GST_RTP_HDREXT_BASE GST_RTP_HDREXT_NTP_64); + /* reset packet spacing */ + priv->packet_spacing = 0; + priv->prev_packet_spacing = GST_CLOCK_TIME_NONE; + priv->ips_rtptime = -1; + return TRUE; /* ERRORS */ @@ -1732,8 +1803,8 @@ gst_rtp_jitter_buffer_flush_stop (GstRtpJitterBuffer * jitterbuffer) priv->next_seqnum = -1; priv->seqnum_base = -1; priv->ips_rtptime = -1; - priv->ips_pts = GST_CLOCK_TIME_NONE; priv->packet_spacing = 0; + priv->prev_packet_spacing = GST_CLOCK_TIME_NONE; priv->next_in_seqnum = -1; priv->clock_rate = -1; priv->ntp64_ext_id = 0; @@ -1744,7 +1815,15 @@ gst_rtp_jitter_buffer_flush_stop (GstRtpJitterBuffer * jitterbuffer) priv->last_elapsed = 0; priv->ext_timestamp = -1; priv->avg_jitter = 0; + priv->min_jitter = G_MAXINT64; priv->last_dts = -1; + priv->avg_clock_diff = -1; + priv->instant_max_jitter_top = 0; + priv->instant_max_jitter_bottom = 0; + priv->etimatied_max_jitter_top = 0; + priv->etimatied_max_jitter_bottom = 0; + priv->dencity_ratio = 0.5f; + priv->estimated_latency_ms = 0; priv->last_rtptime = -1; priv->last_ntpnstime = -1; priv->last_known_ext_rtptime = -1; @@ -1952,7 +2031,7 @@ queue_event (GstRtpJitterBuffer * jitterbuffer, GstEvent * event) GstCaps *caps; gst_event_parse_caps (event, &caps); - gst_jitter_buffer_sink_parse_caps (jitterbuffer, caps, -1); + gst_jitter_buffer_sink_parse_caps (jitterbuffer, caps); break; } case GST_EVENT_SEGMENT: @@ -2124,7 +2203,7 @@ gst_rtp_jitter_buffer_get_clock_rate (GstRtpJitterBuffer * jitterbuffer, if (!caps) goto no_caps; - res = gst_jitter_buffer_sink_parse_caps (jitterbuffer, caps, pt); + res = gst_jitter_buffer_sink_parse_caps (jitterbuffer, caps); gst_caps_unref (caps); if (G_UNLIKELY (!res)) @@ -2519,42 +2598,38 @@ update_rtx_timers (GstRtpJitterBuffer * jitterbuffer, guint16 seqnum, } static void -calculate_packet_spacing (GstRtpJitterBuffer * jitterbuffer, guint32 rtptime, - GstClockTime pts) +calculate_packet_spacing (GstRtpJitterBuffer * jitterbuffer, guint32 rtptime) { GstRtpJitterBufferPrivate *priv = jitterbuffer->priv; /* we need consecutive seqnums with a different * rtptime to estimate the packet spacing. */ if (priv->ips_rtptime != rtptime) { - /* rtptime changed, check pts diff */ - if (priv->ips_pts != -1 && pts != -1 && pts > priv->ips_pts) { - GstClockTime new_packet_spacing = pts - priv->ips_pts; - GstClockTime old_packet_spacing = priv->packet_spacing; - - /* Biased towards bigger packet spacings to prevent - * too many unneeded retransmission requests for next - * packets that just arrive a little later than we would - * expect */ - if (old_packet_spacing > new_packet_spacing) - priv->packet_spacing = - (new_packet_spacing + 3 * old_packet_spacing) / 4; - else if (old_packet_spacing > 0) - priv->packet_spacing = - (3 * new_packet_spacing + old_packet_spacing) / 4; - else - priv->packet_spacing = new_packet_spacing; + /* rtptime changed */ + if (priv->clock_rate > 0 && rtptime > priv->ips_rtptime) { + guint64 rtpdiff; + GstClockTime new_packet_spacing; + + rtpdiff = rtptime - (guint32) priv->ips_rtptime; + new_packet_spacing = + gst_util_uint64_scale_int (rtpdiff, GST_SECOND, priv->clock_rate); + + if (new_packet_spacing != priv->packet_spacing) { + if (priv->prev_packet_spacing == new_packet_spacing + || !GST_CLOCK_TIME_IS_VALID (priv->prev_packet_spacing)) { + /* we received two consecutive packets with the same spacing, update */ + GST_DEBUG_OBJECT (jitterbuffer, + "new packet spacing %" GST_TIME_FORMAT + " old packet spacing %" GST_TIME_FORMAT, + GST_TIME_ARGS (new_packet_spacing), + GST_TIME_ARGS (priv->packet_spacing)); - GST_DEBUG_OBJECT (jitterbuffer, - "new packet spacing %" GST_TIME_FORMAT - " old packet spacing %" GST_TIME_FORMAT - " combined to %" GST_TIME_FORMAT, - GST_TIME_ARGS (new_packet_spacing), - GST_TIME_ARGS (old_packet_spacing), - GST_TIME_ARGS (priv->packet_spacing)); + priv->packet_spacing = new_packet_spacing; + } + priv->prev_packet_spacing = new_packet_spacing; + } } priv->ips_rtptime = rtptime; - priv->ips_pts = pts; } } @@ -2788,12 +2863,112 @@ gst_rtp_jitter_buffer_handle_missing_packets (GstRtpJitterBuffer * jitterbuffer, } } +/* Usually, the distribution of 'jitter' is non-symmetric across the mean value + * (it is similar to the gamma distribution) so the latency estimation based + * on six jitter standard deviations doesn't always give a good prediction. + * The purpose is to give a more precise latency estimation taking into + * account the asymmetric nature of the jitter. To do so we estimate two + * parameters - "etimatied_max_jitter_top", and "etimatied_max_jitter_bottom", + * each of them estimates maximum jitter above or below the mean. In sum, they + * give the total estimated latency. + * The idea is to outline the maximum observed jitters and interpolate them to + * the future. To do so we have two estimations "instant_max_jitter" and "etimatied_max_jitter". + * Both of them are increasing to the observed jitter value if it is bigger than + * their current value. Otherwise, they are decreasing. The "instant_max_jitter" + * is getting decreased to the observed jitter value with some linear factor. + * The "etimatied_max_jitter" is decreasing towards the "instant_max_jitter" + * with some non-linear factor. Non-linearity is added to smooth out the temporal + * fluctuations in "instant_max_jitter". + * */ +static inline void +max_jitter_estimation (guint64 jitter, guint64 * short_estimation, + guint64 * long_estimation, gfloat decrease_factor) +{ + const gint fast_decrease_factor = 64; + const gint slow_decrease_factor = 256; + gint64 diff = 0; + if (jitter >= *short_estimation) + *short_estimation = jitter; + else { + *short_estimation -= ((*short_estimation - jitter) / fast_decrease_factor); + } + diff = *long_estimation - *short_estimation; + if (diff <= 0) + *long_estimation = *short_estimation; + else { + guint delta = diff / slow_decrease_factor; + gfloat ratio = + 1.f - ((gfloat) (*short_estimation) / (gfloat) (*long_estimation)); + *long_estimation -= delta * pow (ratio, (16.f * decrease_factor)); + } +} + +static void +estimate_latency (GstRtpJitterBuffer * jitterbuffer, GstClockTime dts, + guint32 rtptime) +{ + gint64 jitter = 0; + GstClockTimeDiff clock_diff = 0; + GstRtpJitterBufferPrivate *priv = jitterbuffer->priv; + + if (G_UNLIKELY (dts == GST_CLOCK_TIME_NONE) || priv->clock_rate <= 0) + return; + + /* clock_diff - consists Cd + D + J where: + * Cd - initial clock difference between sender and recover + * D - constant delay. + * J - jitter */ + clock_diff = + ((gint64) dts - (gint64) gst_util_uint64_scale_int (rtptime, GST_SECOND, + priv->clock_rate)); + if (priv->avg_clock_diff == -1) { + priv->avg_clock_diff = clock_diff; + return; + } + + /* By averaging clock_diff we getting (Cd + D) + Jm + * where 'Jm' is mean of the jitter */ + priv->avg_clock_diff -= + ((priv->avg_clock_diff - clock_diff) / (gint) MIN (128, MAX (1, + priv->jitter_count))); + + /* Jitter is a deviation form the average clock_diff. */ + jitter = clock_diff - priv->avg_clock_diff; + + /* Purpouse of 'dencity_ratio' is to represent assimmetric distrebution of the jitter. + * The bigger 'dencity_ratio' the faster decreasing the bottom jitter estimation, + * but top estimation slower. If it 0.5 both top and bottom has equal decreasing factor. */ + priv->dencity_ratio -= + ((priv->dencity_ratio - (gfloat) (clock_diff < + priv->avg_clock_diff)) / 16.f); + priv->dencity_ratio = MIN (1., MAX (0., priv->dencity_ratio)); + + /* Update estimations of max jiiter above the jitter mean. */ + max_jitter_estimation (MAX (0, jitter), &priv->instant_max_jitter_top, + &priv->etimatied_max_jitter_top, priv->dencity_ratio); + + /* Update estimations of max jiiter below the jitter mean. */ + max_jitter_estimation (ABS (MIN (0, jitter)), + &priv->instant_max_jitter_bottom, &priv->etimatied_max_jitter_bottom, + (1.f - priv->dencity_ratio)); + + priv->estimated_latency_ms = + (priv->etimatied_max_jitter_top + + priv->etimatied_max_jitter_bottom) / GST_MSECOND; + + GST_LOG_OBJECT (jitterbuffer, + "estimated_latency:%" GST_TIME_FORMAT, + GST_TIME_ARGS (priv->etimatied_max_jitter_top + + priv->etimatied_max_jitter_bottom)); +} + static void calculate_jitter (GstRtpJitterBuffer * jitterbuffer, GstClockTime dts, guint32 rtptime) { - gint32 rtpdiff; - GstClockTimeDiff dtsdiff, rtpdiffns, diff; + gint32 rtpdiff = 0; + GstClockTimeDiff rtpdiffns = 0, diff = 0, dtsdiff = 0; + gint factor = 1; GstRtpJitterBufferPrivate *priv; priv = jitterbuffer->priv; @@ -2801,15 +2976,12 @@ calculate_jitter (GstRtpJitterBuffer * jitterbuffer, GstClockTime dts, if (G_UNLIKELY (dts == GST_CLOCK_TIME_NONE) || priv->clock_rate <= 0) goto no_time; - if (priv->last_dts != -1) - dtsdiff = dts - priv->last_dts; - else - dtsdiff = 0; + /* we need at least two measurements to calculate jitter */ + if (priv->last_dts == -1 && priv->last_rtptime == -1) + goto done; - if (priv->last_rtptime != -1) - rtpdiff = rtptime - (guint32) priv->last_rtptime; - else - rtpdiff = 0; + dtsdiff = dts - priv->last_dts; + rtpdiff = rtptime - (guint32) priv->last_rtptime; /* Guess whether stream currently uses equidistant packet spacing. If we * often see identical timestamps it means the packets are not @@ -2820,8 +2992,9 @@ calculate_jitter (GstRtpJitterBuffer * jitterbuffer, GstClockTime dts, priv->equidistant += 1; priv->equidistant = CLAMP (priv->equidistant, -7, 7); - priv->last_dts = dts; - priv->last_rtptime = rtptime; + /* Non-equidistant packets are not used in jitter calculation */ + if (rtpdiff == 0) + return; if (rtpdiff > 0) rtpdiffns = @@ -2832,8 +3005,15 @@ calculate_jitter (GstRtpJitterBuffer * jitterbuffer, GstClockTime dts, diff = ABS (dtsdiff - rtpdiffns); + priv->jitter_count++; + priv->jitter_sum += diff; + priv->total_avg_jitter = priv->jitter_sum / priv->jitter_count; + priv->max_jitter = MAX (priv->max_jitter, diff); + priv->min_jitter = MIN (priv->min_jitter, diff); + /* jitter is stored in nanoseconds */ - priv->avg_jitter = (diff + (15 * priv->avg_jitter)) >> 4; + factor = MIN (priv->jitter_count, 128); + priv->avg_jitter = (diff + ((factor - 1) * priv->avg_jitter)) / factor; GST_LOG_OBJECT (jitterbuffer, "dtsdiff %" GST_STIME_FORMAT " rtptime %" GST_STIME_FORMAT @@ -2841,6 +3021,10 @@ calculate_jitter (GstRtpJitterBuffer * jitterbuffer, GstClockTime dts, GST_STIME_ARGS (dtsdiff), GST_STIME_ARGS (rtpdiffns), priv->clock_rate, GST_STIME_ARGS (diff), GST_TIME_ARGS (priv->avg_jitter)); +done: + priv->last_dts = dts; + priv->last_rtptime = rtptime; + return; /* ERRORS */ @@ -2993,6 +3177,9 @@ gst_rtp_jitter_buffer_reset (GstRtpJitterBuffer * jitterbuffer, priv->next_seqnum = seqnum; } + GST_DEBUG_OBJECT (jitterbuffer, "setting next_seqnum to #%u", + priv->next_seqnum); + priv->last_in_pts = -1; priv->next_in_seqnum = -1; @@ -3009,13 +3196,10 @@ gst_rtp_jitter_buffer_reset (GstRtpJitterBuffer * jitterbuffer, /* reset spacing estimation when gap */ priv->ips_rtptime = -1; - priv->ips_pts = GST_CLOCK_TIME_NONE; buffers = g_list_copy (priv->gap_packets.head); g_queue_clear (&priv->gap_packets); - priv->ips_rtptime = -1; - priv->ips_pts = GST_CLOCK_TIME_NONE; JBUF_UNLOCK (jitterbuffer->priv); for (l = buffers; l; l = l->next) { @@ -3120,6 +3304,7 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, gint32 packet_rate, max_dropout, max_misorder; RtpTimer *timer = NULL; gboolean is_rtx; + gboolean is_ulpfec; jitterbuffer = GST_RTP_JITTER_BUFFER_CAST (parent); @@ -3136,6 +3321,7 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, gst_rtp_buffer_unmap (&rtp); is_rtx = GST_BUFFER_IS_RETRANSMISSION (buffer); + is_ulpfec = GST_BUFFER_IS_ULPFEC (buffer); now = get_current_running_time (jitterbuffer); /* make sure we have PTS and DTS set */ @@ -3169,9 +3355,9 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, GST_DEBUG_OBJECT (jitterbuffer, "Received packet #%d at time %" GST_TIME_FORMAT - ", discont %d, rtx %d, inband NTP time %" GST_TIME_FORMAT, seqnum, - GST_TIME_ARGS (dts), GST_BUFFER_IS_DISCONT (buffer), is_rtx, - GST_TIME_ARGS (inband_ntp_time)); + ", discont %d, rtx %d ulpfec %d, inband NTP time %" GST_TIME_FORMAT, + seqnum, GST_TIME_ARGS (dts), GST_BUFFER_IS_DISCONT (buffer), is_rtx, + is_ulpfec, GST_TIME_ARGS (inband_ntp_time)); JBUF_LOCK_CHECK (priv, out_flushing); @@ -3191,7 +3377,7 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, /* Try to get the clock-rate from the caps first if we can. If there are no * caps we must fire the signal to get the clock-rate. */ if ((caps = gst_pad_get_current_caps (pad))) { - gst_jitter_buffer_sink_parse_caps (jitterbuffer, caps, pt); + gst_jitter_buffer_sink_parse_caps (jitterbuffer, caps); gst_caps_unref (caps); } } @@ -3202,8 +3388,12 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, pt) == GST_FLOW_FLUSHING) goto out_flushing; - if (G_UNLIKELY (priv->clock_rate == -1)) + if (G_UNLIKELY (priv->clock_rate == -1)) { + priv->no_clock_rate_count++; goto no_clock_rate; + } else { + priv->no_clock_rate_count = 0; + } gst_rtp_packet_rate_ctx_reset (&priv->packet_rate_ctx, priv->clock_rate); priv->last_known_ext_rtptime = -1; @@ -3222,8 +3412,10 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, if (G_UNLIKELY (priv->eos)) goto have_eos; - if (!is_rtx) + if (!is_rtx && !is_ulpfec) { calculate_jitter (jitterbuffer, dts, rtptime); + estimate_latency (jitterbuffer, dts, rtptime); + } if (priv->seqnum_base != -1) { gint gap; @@ -3247,7 +3439,7 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, expected = priv->next_in_seqnum; /* don't update packet-rate based on RTX, as those arrive highly unregularly */ - if (!is_rtx) { + if (!is_rtx && !is_ulpfec) { packet_rate = gst_rtp_packet_rate_ctx_update (&priv->packet_rate_ctx, seqnum, rtptime); GST_TRACE_OBJECT (jitterbuffer, "updated packet_rate: %d", packet_rate); @@ -3299,7 +3491,6 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, do_next_seqnum = TRUE; /* take rtptime and pts to calculate packet spacing */ priv->ips_rtptime = rtptime; - priv->ips_pts = pts; } else { gint gap; @@ -3308,23 +3499,9 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, GST_DEBUG_OBJECT (jitterbuffer, "expected #%d, got #%d, gap of %d", expected, seqnum, gap); - if (G_UNLIKELY (gap > 0 && - rtp_timer_queue_length (priv->timers) >= max_dropout)) { - /* If we have timers for more than RTP_MAX_DROPOUT packets - * pending this means that we have a huge gap overall. We can - * reset the jitterbuffer at this point because there's - * just too much data missing to be able to do anything - * sensible with the past data. Just try again from the - * next packet */ - GST_WARNING_OBJECT (jitterbuffer, "%d pending timers > %d - resetting", - rtp_timer_queue_length (priv->timers), max_dropout); - g_queue_insert_sorted (&priv->gap_packets, buffer, - (GCompareDataFunc) compare_buffer_seqnum, NULL); - return gst_rtp_jitter_buffer_reset (jitterbuffer, pad, parent, seqnum); - } - /* Special handling of large gaps */ - if (!is_rtx && ((gap != -1 && gap < -max_misorder) || (gap >= max_dropout))) { + if (!is_rtx && !is_ulpfec && ((gap != -1 && gap < -max_misorder) + || (gap >= max_dropout))) { gboolean reset = handle_big_gap_buffer (jitterbuffer, buffer, pt, seqnum, gap, max_dropout, max_misorder); if (reset) { @@ -3346,16 +3523,20 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, pts = rtp_jitter_buffer_calculate_pts (priv->jbuf, dts, estimated_dts, rtptime, gst_element_get_base_time (GST_ELEMENT_CAST (jitterbuffer)), - gap, is_rtx, &ntp_time); + gap, is_rtx || is_ulpfec, &ntp_time); if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (pts))) { - /* A valid timestamp cannot be calculated, discard packet */ - goto discard_invalid; + if (is_ulpfec) { + pts = priv->ips_rtptime; + } else { + /* A valid timestamp cannot be calculated, discard packet */ + goto discard_invalid; + } } if (G_LIKELY (gap == 0)) { /* packet is expected */ - calculate_packet_spacing (jitterbuffer, rtptime, pts); + calculate_packet_spacing (jitterbuffer, rtptime); do_next_seqnum = TRUE; } else { @@ -3383,7 +3564,6 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, /* reset spacing estimation when gap */ priv->ips_rtptime = -1; - priv->ips_pts = GST_CLOCK_TIME_NONE; } } @@ -3591,8 +3771,11 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, } no_clock_rate: { - GST_WARNING_OBJECT (jitterbuffer, - "No clock-rate in caps!, dropping buffer"); + if (((priv->no_clock_rate_count) % NO_CLOCK_RATE_WARNING_THROTTLE) == 1) { + GST_WARNING_OBJECT (jitterbuffer, + "No clock-rate in caps!, dropping buffer (count=%d)", + priv->no_clock_rate_count); + } gst_buffer_unref (buffer); goto finished; } @@ -3646,8 +3829,8 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, discard_invalid: { GST_DEBUG_OBJECT (jitterbuffer, - "cannot calculate a valid pts for #%d (rtx: %d), discard", - seqnum, is_rtx); + "cannot calculate a valid pts for #%d (rtx: %d ulpfec: %d), discard", + seqnum, is_rtx, is_ulpfec); gst_buffer_unref (buffer); goto finished; } @@ -4729,7 +4912,7 @@ gst_rtp_jitter_buffer_chain_rtcp (GstPad * pad, GstObject * parent, GstRtpJitterBuffer *jitterbuffer; GstRtpJitterBufferPrivate *priv; GstFlowReturn ret = GST_FLOW_OK; - guint32 ssrc; + guint32 ssrc = 0x0; GstRTCPPacket packet; guint64 ext_rtptime, ntptime; GstClockTime ntpnstime = GST_CLOCK_TIME_NONE; @@ -4820,9 +5003,13 @@ gst_rtp_jitter_buffer_chain_rtcp (GstPad * pad, GstObject * parent, goto empty_buffer; JBUF_LOCK (priv); - if (cname) - insert_cname_ssrc_mapping (jitterbuffer, cname, ssrc); - + if (cname) { + if (g_utf8_validate (cname, -1, NULL)) { + insert_cname_ssrc_mapping (jitterbuffer, cname, ssrc); + } else { + GST_WARNING ("ignoring CNAME with non-utf8 data %s", cname); + } + } /* convert the RTP timestamp to our extended timestamp, using the same offset * we used in the jitterbuffer */ ext_rtptime = priv->jbuf->ext_rtptime; @@ -5069,6 +5256,10 @@ gst_rtp_jitter_buffer_set_property (GObject * object, gst_element_post_message (GST_ELEMENT_CAST (jitterbuffer), gst_message_new_latency (GST_OBJECT_CAST (jitterbuffer))); + + JBUF_LOCK (priv); + queue_event (jitterbuffer, gst_event_new_latency_changed ()); + JBUF_UNLOCK (priv); } break; } @@ -5390,16 +5581,21 @@ gst_rtp_jitter_buffer_create_stats (GstRtpJitterBuffer * jbuf) GstStructure *s; JBUF_LOCK (priv); - s = gst_structure_new ("application/x-rtp-jitterbuffer-stats", - "num-pushed", G_TYPE_UINT64, priv->num_pushed, - "num-lost", G_TYPE_UINT64, priv->num_lost, - "num-late", G_TYPE_UINT64, priv->num_late, - "num-duplicates", G_TYPE_UINT64, priv->num_duplicates, - "avg-jitter", G_TYPE_UINT64, priv->avg_jitter, - "rtx-count", G_TYPE_UINT64, priv->num_rtx_requests, - "rtx-success-count", G_TYPE_UINT64, priv->num_rtx_success, - "rtx-per-packet", G_TYPE_DOUBLE, priv->avg_rtx_num, - "rtx-rtt", G_TYPE_UINT64, priv->avg_rtx_rtt, NULL); + s = gst_structure_new_id (quark_application_x_rtp_jitterbuffer_stats, + quark_num_pushed, G_TYPE_UINT64, priv->num_pushed, + quark_num_lost, G_TYPE_UINT64, priv->num_lost, + quark_num_late, G_TYPE_UINT64, priv->num_late, + quark_num_duplicates, G_TYPE_UINT64, priv->num_duplicates, + quark_avg_jitter, G_TYPE_UINT64, priv->avg_jitter, + quark_total_avg_jitter, G_TYPE_UINT64, priv->total_avg_jitter, + quark_min_jitter, G_TYPE_UINT64, priv->min_jitter, + quark_max_jitter, G_TYPE_UINT64, priv->max_jitter, + quark_estimated_latency_ms, G_TYPE_UINT, priv->estimated_latency_ms, + quark_latency_ms, G_TYPE_UINT, priv->latency_ms, + quark_rtx_count, G_TYPE_UINT64, priv->num_rtx_requests, + quark_rtx_success_count, G_TYPE_UINT64, priv->num_rtx_success, + quark_rtx_per_packet, G_TYPE_DOUBLE, priv->avg_rtx_num, + quark_rtx_rtt, G_TYPE_UINT64, priv->avg_rtx_rtt, NULL); JBUF_UNLOCK (priv); return s; diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmanager.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmanager.c index 59a9eeb3a2c..d93aa310417 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmanager.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmanager.c @@ -40,6 +40,7 @@ #include "gstrtphdrext-ntp.h" #include "gstrtphdrext-repairedstreamid.h" #include "gstrtphdrext-streamid.h" +#include "gstrtphdrext-roi.h" static gboolean plugin_init (GstPlugin * plugin) @@ -65,6 +66,7 @@ plugin_init (GstPlugin * plugin) ret |= GST_ELEMENT_REGISTER (rtphdrextntp64, plugin); ret |= GST_ELEMENT_REGISTER (rtphdrextstreamid, plugin); ret |= GST_ELEMENT_REGISTER (rtphdrextrepairedstreamid, plugin); + ret |= GST_ELEMENT_REGISTER (rtphdrextroi, plugin); return ret; } diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmux.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmux.c index a5ad2306cb3..89ca7cdfde6 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmux.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpmux.c @@ -967,7 +967,7 @@ gst_rtp_mux_ready_to_paused (GstRTPMux * rtp_mux) rtp_mux->send_stream_start = TRUE; if (rtp_mux->seqnum_offset == -1) - rtp_mux->seqnum_base = g_random_int_range (0, G_MAXUINT16); + rtp_mux->seqnum_base = g_random_int_range (0, G_MAXUINT16 / 2); else rtp_mux->seqnum_base = rtp_mux->seqnum_offset; rtp_mux->seqnum = rtp_mux->seqnum_base; diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpptdemux.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpptdemux.c index 0454d971348..6f6273e682c 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpptdemux.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpptdemux.c @@ -641,27 +641,6 @@ gst_rtp_pt_demux_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) res = TRUE; break; } - case GST_EVENT_CUSTOM_DOWNSTREAM: - { - const GstStructure *s; - - s = gst_event_get_structure (event); - - if (gst_structure_has_name (s, "GstRTPPacketLost")) { - GstPad *srcpad = find_pad_for_pt (rtpdemux, rtpdemux->last_pt); - - if (srcpad) { - res = gst_pad_push_event (srcpad, event); - gst_object_unref (srcpad); - } else { - gst_event_unref (event); - } - - } else { - res = gst_pad_event_default (pad, parent, event); - } - break; - } default: res = gst_pad_event_default (pad, parent, event); break; diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxreceive.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxreceive.c index 5d74c235031..b64c12b0bac 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxreceive.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxreceive.c @@ -728,7 +728,12 @@ _gst_rtp_buffer_new_from_rtx (GstRtpRtxReceive * rtx, GstRTPBuffer * rtp, /* copy extension if any */ if (rtp->size[1]) { - mem = rewrite_header_extensions (rtx, rtp); + if (rtx->rid_stream) { + mem = rewrite_header_extensions (rtx, rtp); + } else { + mem = gst_memory_copy (rtp->map[1].memory, + (guint8 *) rtp->data[1] - rtp->map[1].data, rtp->size[1]); + } gst_buffer_append_memory (new_buffer, mem); } diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c index 1c36e5272f2..55be31cae31 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.c @@ -45,13 +45,22 @@ #include #include "gstrtprtxsend.h" +#include "rtpstats.h" GST_DEBUG_CATEGORY_STATIC (gst_rtp_rtx_send_debug); #define GST_CAT_DEFAULT gst_rtp_rtx_send_debug +#define MIN_RTP_HEADER_LEN 12 + +#define UNLIMITED_KBPS (-1) + #define DEFAULT_RTX_PAYLOAD_TYPE 0 #define DEFAULT_MAX_SIZE_TIME 0 #define DEFAULT_MAX_SIZE_PACKETS 100 +#define DEFAULT_MAX_KBPS UNLIMITED_KBPS +#define DEFAULT_MAX_BUCKET_SIZE UNLIMITED_KBPS +#define DEFAULT_STUFFING_KBPS 0 +#define DEFAULT_STUFFING_MAX_BURST_PACKETS 20 enum { @@ -63,6 +72,11 @@ enum PROP_NUM_RTX_REQUESTS, PROP_NUM_RTX_PACKETS, PROP_CLOCK_RATE_MAP, + PROP_MAX_KBPS, + PROP_MAX_BUCKET_SIZE, + PROP_STUFFING_KBPS, + PROP_STUFFING_MAX_BURST_PACKETS, + PROP_LAST, }; enum @@ -156,7 +170,11 @@ G_DEFINE_TYPE_WITH_CODE (GstRtpRtxSend, gst_rtp_rtx_send, GST_TYPE_ELEMENT, GST_ELEMENT_REGISTER_DEFINE (rtprtxsend, "rtprtxsend", GST_RANK_NONE, GST_TYPE_RTP_RTX_SEND); +static void gst_rtp_rtx_send_reset_stuffing_bucket (GstRtpRtxSend * rtx, + gint kbps); + #define IS_RTX_ENABLED(rtx) (g_hash_table_size ((rtx)->rtx_pt_map) > 0) +#define RTX_OVERHEAD 2 typedef struct { @@ -174,6 +192,7 @@ buffer_queue_item_free (BufferQueueItem * item) typedef struct { + guint32 ssrc; guint32 rtx_ssrc; guint16 seqnum_base, next_seqnum; gint clock_rate; @@ -183,10 +202,11 @@ typedef struct } SSRCRtxData; static SSRCRtxData * -ssrc_rtx_data_new (guint32 rtx_ssrc) +ssrc_rtx_data_new (guint32 ssrc, guint32 rtx_ssrc) { SSRCRtxData *data = g_new0 (SSRCRtxData, 1); + data->ssrc = ssrc; data->rtx_ssrc = rtx_ssrc; data->next_seqnum = data->seqnum_base = g_random_int_range (0, G_MAXUINT16); data->queue = g_sequence_new ((GDestroyNotify) buffer_queue_item_free); @@ -201,6 +221,12 @@ ssrc_rtx_data_free (SSRCRtxData * data) g_free (data); } +static gint +get_buffer_bytes_size (GstBuffer * buffer) +{ + return gst_buffer_get_size (buffer) + UDP_IP_HEADER_OVERHEAD; +} + typedef enum { RTX_TASK_START, @@ -337,6 +363,62 @@ gst_rtp_rtx_send_class_init (GstRtpRtxSendClass * klass) G_CALLBACK (gst_rtp_rtx_send_clear_extensions), NULL, NULL, NULL, G_TYPE_NONE, 0); + /** + * rtprtxsend:max-kbps: + * + * The maximum number of kilobits of RTX packets to allow. + * + * Since: 1.22 + */ + g_object_class_install_property (gobject_class, PROP_MAX_KBPS, + g_param_spec_int ("max-kbps", "Maximum Kbps", + "The maximum number of kilobits of RTX packets to allow " + "(-1 = unlimited)", -1, G_MAXINT, DEFAULT_MAX_KBPS, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + + /** + * rtprtxsend:max-bucket-size: + * + * The size of the token bucket, related to burstiness resilience. + * + * Since: 1.22 + */ + g_object_class_install_property (gobject_class, PROP_MAX_BUCKET_SIZE, + g_param_spec_int ("max-bucket-size", "Maximum Bucket Size (Kb)", + "The size of the token bucket, related to burstiness resilience " + "(-1 = unlimited)", -1, G_MAXINT, DEFAULT_MAX_BUCKET_SIZE, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + + /** + * rtprtxsend:stuffing-kbps: + * + * Use RTX to send stuffing packets to produce a near constant rate of + * total packets (media and stuffing) + * + * Since: 1.22 + */ + g_object_class_install_property (gobject_class, PROP_STUFFING_KBPS, + g_param_spec_int ("stuffing-kbps", "Stuffing kbps", + "Use RTX to send stuffing packets to produce a near constant rate of " + "total packets (media and stuffing) (0 = disabled)", + 0, G_MAXINT, DEFAULT_STUFFING_KBPS, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + + /** + * rtprtxsend:stuffing-max-burst-packets: + * + * The maximum amount of stuffing-packets to produce in a single burst. + * + * Since: 1.22 + */ + g_object_class_install_property (gobject_class, + PROP_STUFFING_MAX_BURST_PACKETS, + g_param_spec_int ("stuffing-max-burst-packets", + "Stuffing Max Burst Packets", + "The maximum amount of stuffing-packets to produce in a single burst", + 0, G_MAXINT, DEFAULT_STUFFING_MAX_BURST_PACKETS, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + gst_element_class_add_static_pad_template (gstelement_class, &src_factory); gst_element_class_add_static_pad_template (gstelement_class, &sink_factory); @@ -427,6 +509,8 @@ gst_rtp_rtx_send_init (GstRtpRtxSend * rtx) rtx->max_size_packets = DEFAULT_MAX_SIZE_PACKETS; rtx->dummy_writable = gst_buffer_new (); + gst_rtp_rtx_send_reset_stuffing_bucket (rtx, DEFAULT_STUFFING_KBPS); + rtx->last_stuffing_ssrc = -1; } static gboolean @@ -458,6 +542,8 @@ gst_rtp_rtx_send_push_out (GstRtpRtxSend * rtx, gpointer object) data->visible = TRUE; data->destroy = gst_rtp_rtx_data_queue_item_free; + GST_LOG_OBJECT (rtx, "Adding %" GST_PTR_FORMAT, object); + success = gst_data_queue_push (rtx->queue, data); if (!success) @@ -498,7 +584,7 @@ gst_rtp_rtx_send_get_ssrc_data (GstRtpRtxSend * rtx, guint32 ssrc) g_free (ssrc_str); } rtx_ssrc = gst_rtp_rtx_send_choose_ssrc (rtx, rtx_ssrc, consider); - data = ssrc_rtx_data_new (rtx_ssrc); + data = ssrc_rtx_data_new (ssrc, rtx_ssrc); g_hash_table_insert (rtx->ssrc_data, GUINT_TO_POINTER (ssrc), data); g_hash_table_insert (rtx->rtx_ssrcs, GUINT_TO_POINTER (rtx_ssrc), GUINT_TO_POINTER (ssrc)); @@ -668,7 +754,7 @@ rewrite_header_extensions (GstRtpRtxSend * rtx, GstRTPBuffer * rtp) * Copy memory to avoid to manually copy each rtp buffer field. */ static GstBuffer * -gst_rtp_rtx_buffer_new (GstRtpRtxSend * rtx, GstBuffer * buffer) +gst_rtp_rtx_buffer_new (GstRtpRtxSend * rtx, GstBuffer * buffer, guint8 padlen) { GstMemory *mem = NULL; GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; @@ -704,7 +790,12 @@ gst_rtp_rtx_buffer_new (GstRtpRtxSend * rtx, GstBuffer * buffer) /* copy extension if any */ if (rtp.size[1]) { - mem = rewrite_header_extensions (rtx, &rtp); + if (rtx->rid_stream) { + mem = rewrite_header_extensions (rtx, &rtp); + } else { + mem = gst_memory_copy (rtp.map[1].memory, + (guint8 *) rtp.data[1] - rtp.map[1].data, rtp.size[1]); + } gst_buffer_append_memory (new_buffer, mem); } @@ -719,6 +810,15 @@ gst_rtp_rtx_buffer_new (GstRtpRtxSend * rtx, GstBuffer * buffer) gst_memory_unmap (mem, &map); gst_buffer_append_memory (new_buffer, mem); + if (padlen) { + mem = gst_allocator_alloc (NULL, padlen, NULL); + gst_memory_map (mem, &map, GST_MAP_WRITE); + memset (map.data, 0, padlen - 1); + map.data[padlen - 1] = padlen; + gst_memory_unmap (mem, &map); + gst_buffer_append_memory (new_buffer, mem); + } + /* everything needed is copied */ gst_rtp_buffer_unmap (&rtp); @@ -728,7 +828,7 @@ gst_rtp_rtx_buffer_new (GstRtpRtxSend * rtx, GstBuffer * buffer) gst_rtp_buffer_set_seq (&new_rtp, seqnum); gst_rtp_buffer_set_payload_type (&new_rtp, fmtp); /* RFC 4588: let other elements do the padding, as normal */ - gst_rtp_buffer_set_padding (&new_rtp, FALSE); + gst_rtp_buffer_set_padding (&new_rtp, padlen != 0); gst_rtp_buffer_unmap (&new_rtp); /* Copy over timestamps */ @@ -750,6 +850,28 @@ buffer_queue_items_cmp (BufferQueueItem * a, BufferQueueItem * b, return gst_rtp_buffer_compare_seqnum (b->seqnum, a->seqnum); } +static gboolean +gst_rtp_rtx_send_token_bucket (GstRtpRtxSend * rtx, GstBuffer * buf) +{ + GstClock *clock; + guint64 tokens; + + /* with an unlimited bucket-size, we have nothing to do */ + if (rtx->max_bucket_size == UNLIMITED_KBPS || rtx->max_kbps == UNLIMITED_KBPS) + return TRUE; + + /* without a clock, nothing to do */ + clock = GST_ELEMENT_CAST (rtx)->clock; + if (clock == NULL) { + GST_WARNING_OBJECT (rtx, "No clock, can't get the time"); + return TRUE; + } + + tokens = get_buffer_bytes_size (buf) * 8; + token_bucket_add_tokens (&rtx->max_tb, gst_clock_get_time (clock)); + return token_bucket_take_tokens (&rtx->max_tb, tokens, FALSE); +} + static gboolean gst_rtp_rtx_send_src_event (GstPad * pad, GstObject * parent, GstEvent * event) { @@ -795,8 +917,13 @@ gst_rtp_rtx_send_src_event (GstPad * pad, GstObject * parent, GstEvent * event) (GCompareDataFunc) buffer_queue_items_cmp, NULL); if (iter) { BufferQueueItem *item = g_sequence_get (iter); - GST_LOG_OBJECT (rtx, "found %u", item->seqnum); - rtx_buf = gst_rtp_rtx_buffer_new (rtx, item->buffer); + GST_LOG_OBJECT (rtx, "found %" G_GUINT16_FORMAT, item->seqnum); + if (gst_rtp_rtx_send_token_bucket (rtx, item->buffer)) { + rtx_buf = gst_rtp_rtx_buffer_new (rtx, item->buffer, 0); + } else { + GST_DEBUG_OBJECT (rtx, "Packet #%" G_GUINT16_FORMAT + " dropped due to full bucket", item->seqnum); + } } #ifndef GST_DISABLE_DEBUG else { @@ -1010,12 +1137,12 @@ gst_rtp_rtx_send_get_ts_diff (SSRCRtxData * data) } /* Must be called with lock */ -static void +static SSRCRtxData * process_buffer (GstRtpRtxSend * rtx, GstBuffer * buffer) { GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; BufferQueueItem *item; - SSRCRtxData *data; + SSRCRtxData *data = NULL; guint16 seqnum; guint8 payload_type; guint32 ssrc, rtptime; @@ -1058,19 +1185,197 @@ process_buffer (GstRtpRtxSend * rtx, GstBuffer * buffer) g_sequence_remove (g_sequence_get_begin_iter (data->queue)); } } + + return data; +} + +static GstFlowReturn +gst_rtp_rtx_send_push (GstRtpRtxSend * rtx, GstBuffer * buffer) +{ + GstFlowReturn ret; + guint buffer_size = get_buffer_bytes_size (buffer); + token_bucket_take_tokens (&rtx->stuff_tb, buffer_size * 8, TRUE); + + GST_OBJECT_UNLOCK (rtx); + GST_LOG_OBJECT (rtx, + "Pushing buffer %" GST_PTR_FORMAT " with network size: %u", buffer, + buffer_size); + ret = gst_pad_push (rtx->srcpad, buffer); + GST_OBJECT_LOCK (rtx); + + return ret; +} + +static guint +gst_rtp_rtx_send_get_stuffing_buffers (GstRtpRtxSend * rtx, + SSRCRtxData * rtx_data, GSequenceIter ** start, GSequenceIter ** end) +{ + GstClockTime running_time; + GstClockTime window_size = 100 * GST_MSECOND; + + GSequenceIter *first = NULL; + GSequenceIter *last = NULL; + GSequenceIter *current; + + guint available_stuffing_bytes = 0; + guint bucket_size_bytes = (guint) (rtx->stuff_tb.bucket_size / 8); + + if (g_sequence_is_empty (rtx_data->queue)) + return 0; + + /* determine the first and last item on the queue to do stuffing with */ + first = g_sequence_get_begin_iter (rtx_data->queue); + last = g_sequence_get_end_iter (rtx_data->queue); + + running_time = + gst_clock_get_time (GST_ELEMENT_CLOCK (rtx)) - + GST_ELEMENT_CAST (rtx)->base_time; + + current = last; + while (current != first) { + BufferQueueItem *item = g_sequence_get (g_sequence_iter_prev (current)); + /* the additional 2 bytes here is when turning this into a RTX buffer */ + guint buffer_size = get_buffer_bytes_size (item->buffer) + 2; + GST_LOG_OBJECT (rtx, "Considering buffer #%u with size %u, total: %u", + item->seqnum, buffer_size, available_stuffing_bytes); + + /* stop here if we will exceed the bucket with this buffer */ + if (available_stuffing_bytes + buffer_size > bucket_size_bytes) + break; + + /* stop here if the packet is "too old", but only if we already have a packet to push */ + if (current != last && GST_BUFFER_PTS_IS_VALID (item->buffer) && + GST_CLOCK_DIFF (GST_BUFFER_PTS (item->buffer), + running_time) > window_size) { + break; + } + + current = g_sequence_iter_prev (current); + available_stuffing_bytes += buffer_size; + } + + /* we were unable to find any buffers */ + if (current == last) + return 0; + + *start = current; + *end = last; + + return available_stuffing_bytes * 8; +} + +static GstFlowReturn +gst_rtp_rtx_send_push_stuffing (GstRtpRtxSend * rtx, SSRCRtxData * rtx_data) +{ + GstFlowReturn ret = GST_FLOW_OK; + GSequenceIter *first = NULL; + GSequenceIter *last = NULL; + GSequenceIter *current; + guint available_stuffing_bits; + gint missing_stuffing_bytes; + guint stuffing_pushed = 0; + + if (rtx->stuff_tb.bucket_size <= 0) { + GST_DEBUG_OBJECT (rtx, "No room in bucket for stuffing"); + return GST_FLOW_OK; + } + + available_stuffing_bits = + gst_rtp_rtx_send_get_stuffing_buffers (rtx, rtx_data, &first, &last); + + /* we cannot generate any stuffing, no bits */ + if (!available_stuffing_bits) { + GST_DEBUG_OBJECT (rtx, "No available RTX packets to use for stuffing"); + return GST_FLOW_OK; + } + + GST_DEBUG_OBJECT (rtx, + "About to produce up to %u bytes of stuffing using %u bytes of packet data", + (guint) (rtx->stuff_tb.bucket_size / 8), available_stuffing_bits / 8); + + missing_stuffing_bytes = + MAX (0, (rtx->stuff_tb.bucket_size - (gint) available_stuffing_bits) / 8); + + current = first; + while (ret == GST_FLOW_OK && (rtx->stuff_tb.bucket_size / 8) > 0 + && stuffing_pushed < rtx->stuffing_max_burst_packets) { + GstBuffer *rtx_buf; + BufferQueueItem *item = g_sequence_get (current); + gsize bufsize = get_buffer_bytes_size (item->buffer) + 2; + guint8 padding = MIN (G_MAXUINT8, missing_stuffing_bytes); + if (padding && bufsize < 300) { /* we only pad buffers less than 300 bytes in size */ + missing_stuffing_bytes -= padding; + GST_LOG_OBJECT (rtx, "adding %u bytes of padding, still missing: %u", + padding, missing_stuffing_bytes); + } + rtx_buf = gst_rtp_rtx_buffer_new (rtx, item->buffer, padding); + ret = gst_rtp_rtx_send_push (rtx, rtx_buf); + + GST_LOG_OBJECT (rtx, + "Pushed 1 stuffing packet with size %u - bucket_size=%d", + (guint) get_buffer_bytes_size (rtx_buf), + (gint) (rtx->stuff_tb.bucket_size / 8)); + current = g_sequence_iter_next (current); + + /* if we are at the end, start over */ + if (current == last) { + current = first; + missing_stuffing_bytes = + MAX (0, + (rtx->stuff_tb.bucket_size - (gint) available_stuffing_bits) / 8); + } + stuffing_pushed++; + } + + return ret; +} + +/* call with LOCK */ +static SSRCRtxData * +gst_rtp_rtx_send_get_rtx_data (GstRtpRtxSend * rtx, GstBuffer * buffer) +{ + SSRCRtxData *rtx_data = process_buffer (rtx, buffer); + + if (rtx_data) { + /* we have rtx data from the current buffer, save the ssrc */ + rtx->last_stuffing_ssrc = rtx_data->ssrc; + return rtx_data; + } + + if (rtx->last_stuffing_ssrc != -1) { + /* we do not have rtx data from the current buffer, so fetch the last one sent */ + return gst_rtp_rtx_send_get_ssrc_data (rtx, rtx->last_stuffing_ssrc); + } + + return NULL; } static GstFlowReturn gst_rtp_rtx_send_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) { - GstRtpRtxSend *rtx = GST_RTP_RTX_SEND_CAST (parent); GstFlowReturn ret; + GstRtpRtxSend *rtx; + SSRCRtxData *rtx_data = NULL; + GstClock *clock; + + rtx = GST_RTP_RTX_SEND_CAST (parent); GST_OBJECT_LOCK (rtx); - if (rtx->rtx_pt_map_structure) - process_buffer (rtx, buffer); + clock = GST_ELEMENT_CLOCK (rtx); + + if (clock) { + token_bucket_add_tokens (&rtx->stuff_tb, gst_clock_get_time (clock)); + } + + if (IS_RTX_ENABLED (rtx)) + rtx_data = gst_rtp_rtx_send_get_rtx_data (rtx, buffer); + + ret = gst_rtp_rtx_send_push (rtx, buffer); + + if (ret == GST_FLOW_OK && clock && rtx_data && rtx->stuffing_kbps != 0) + ret = gst_rtp_rtx_send_push_stuffing (rtx, rtx_data); + GST_OBJECT_UNLOCK (rtx); - ret = gst_pad_push (rtx->srcpad, buffer); return ret; } @@ -1090,7 +1395,16 @@ gst_rtp_rtx_send_chain_list (GstPad * pad, GstObject * parent, GstFlowReturn ret; GST_OBJECT_LOCK (rtx); - gst_buffer_list_foreach (list, process_buffer_from_list, rtx); + if (IS_RTX_ENABLED (rtx)) { + if (rtx->stuffing_kbps != 0) + GST_WARNING_OBJECT (rtx, "RTX stuffing for chain_list not implemented"); + + gst_buffer_list_foreach (list, process_buffer_from_list, rtx); + + GST_OBJECT_UNLOCK (rtx); + gst_rtp_rtx_send_push_out (rtx, list); + return GST_FLOW_OK; + } GST_OBJECT_UNLOCK (rtx); ret = gst_pad_push_list (rtx->srcpad, list); @@ -1110,9 +1424,9 @@ gst_rtp_rtx_send_src_loop (GstRtpRtxSend * rtx) GST_OBJECT_LOCK (rtx); /* Update statistics just before pushing. */ rtx->num_rtx_packets++; + gst_rtp_rtx_send_push (rtx, GST_BUFFER (data->object)); GST_OBJECT_UNLOCK (rtx); - gst_pad_push (rtx->srcpad, GST_BUFFER (data->object)); } else if (GST_IS_EVENT (data->object)) { gst_pad_push_event (rtx->srcpad, GST_EVENT (data->object)); @@ -1192,6 +1506,26 @@ gst_rtp_rtx_send_get_property (GObject * object, g_value_set_boxed (value, rtx->clock_rate_map_structure); GST_OBJECT_UNLOCK (rtx); break; + case PROP_MAX_KBPS: + GST_OBJECT_LOCK (rtx); + g_value_set_int (value, rtx->max_kbps); + GST_OBJECT_UNLOCK (rtx); + break; + case PROP_MAX_BUCKET_SIZE: + GST_OBJECT_LOCK (rtx); + g_value_set_int (value, rtx->max_bucket_size); + GST_OBJECT_UNLOCK (rtx); + break; + case PROP_STUFFING_KBPS: + GST_OBJECT_LOCK (rtx); + g_value_set_int (value, rtx->stuffing_kbps); + GST_OBJECT_UNLOCK (rtx); + break; + case PROP_STUFFING_MAX_BURST_PACKETS: + GST_OBJECT_LOCK (rtx); + g_value_set_int (value, rtx->stuffing_max_burst_packets); + GST_OBJECT_UNLOCK (rtx); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1214,6 +1548,37 @@ structure_to_hash_table (GQuark field_id, const GValue * value, gpointer hash) return TRUE; } +static void +gst_rtp_rtx_send_reset_max_bucket (GstRtpRtxSend * rtx, + gint max_kbps, gint max_bucket_size) +{ + gboolean prev_unlimited = rtx->max_kbps == UNLIMITED_KBPS || + rtx->max_bucket_size == UNLIMITED_KBPS; + gboolean unlimited = max_kbps == UNLIMITED_KBPS || + max_bucket_size == UNLIMITED_KBPS; + + rtx->max_kbps = max_kbps; + rtx->max_bucket_size = max_bucket_size; + + token_bucket_init (&rtx->max_tb, + max_kbps == -1 ? -1 : max_kbps * 1000, + max_bucket_size == -1 ? -1 : max_bucket_size * 1000); + + /* Fill the bucket to max if we switched from unlimited to limited + * kbps mode */ + if (prev_unlimited && !unlimited) { + rtx->max_tb.bucket_size = max_bucket_size * 1000; + } +} + +static void +gst_rtp_rtx_send_reset_stuffing_bucket (GstRtpRtxSend * rtx, gint kbps) +{ + GST_INFO_OBJECT (rtx, "Setting stuffing to %u kbps", kbps); + rtx->stuffing_kbps = kbps; + token_bucket_init (&rtx->stuff_tb, kbps * 1000, kbps * 1000); +} + static void gst_rtp_rtx_send_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) @@ -1234,8 +1599,8 @@ gst_rtp_rtx_send_set_property (GObject * object, gst_structure_free (rtx->rtx_pt_map_structure); rtx->rtx_pt_map_structure = g_value_dup_boxed (value); g_hash_table_remove_all (rtx->rtx_pt_map); - gst_structure_foreach (rtx->rtx_pt_map_structure, structure_to_hash_table, - rtx->rtx_pt_map); + gst_structure_foreach (rtx->rtx_pt_map_structure, + structure_to_hash_table, rtx->rtx_pt_map); GST_OBJECT_UNLOCK (rtx); if (IS_RTX_ENABLED (rtx)) @@ -1264,6 +1629,35 @@ gst_rtp_rtx_send_set_property (GObject * object, structure_to_hash_table, rtx->clock_rate_map); GST_OBJECT_UNLOCK (rtx); break; + case PROP_MAX_KBPS: + GST_OBJECT_LOCK (rtx); + { + gint max_kbps = g_value_get_int (value); + gst_rtp_rtx_send_reset_max_bucket (rtx, max_kbps, rtx->max_bucket_size); + } + GST_OBJECT_UNLOCK (rtx); + break; + case PROP_MAX_BUCKET_SIZE: + GST_OBJECT_LOCK (rtx); + { + gint max_bucket_size = g_value_get_int (value); + gst_rtp_rtx_send_reset_max_bucket (rtx, rtx->max_kbps, max_bucket_size); + } + GST_OBJECT_UNLOCK (rtx); + break; + case PROP_STUFFING_KBPS: + GST_OBJECT_LOCK (rtx); + { + gint kbps = g_value_get_int (value); + gst_rtp_rtx_send_reset_stuffing_bucket (rtx, kbps); + } + GST_OBJECT_UNLOCK (rtx); + break; + case PROP_STUFFING_MAX_BURST_PACKETS: + GST_OBJECT_LOCK (rtx); + rtx->stuffing_max_burst_packets = g_value_get_int (value); + GST_OBJECT_UNLOCK (rtx); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.h b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.h index 60a4ec5bb91..717d8be2ef0 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtprtxsend.h @@ -27,6 +27,7 @@ #include #include #include +#include "tokenbucket.h" G_BEGIN_DECLS @@ -83,6 +84,20 @@ struct _GstRtpRtxSend GstRTPHeaderExtension *rid_repaired; GstBuffer *dummy_writable; + + /* bucket */ + gint max_kbps; + gint max_bucket_size; + TokenBucket max_tb; + + /* stuffing properties */ + gint stuffing_kbps; + guint stuffing_max_burst_packets; + + /* last ssrc used for stuffing */ + gint last_stuffing_ssrc; + + TokenBucket stuff_tb; }; struct _GstRtpRtxSendClass diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c index c50a8072968..1d65fcd9614 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.c @@ -188,11 +188,17 @@ GST_STATIC_PAD_TEMPLATE ("send_rtcp_src", GST_STATIC_CAPS ("application/x-rtcp") ); +static GQuark quark_rtx_count; +static GQuark quark_recv_rtx_req_count; +static GQuark quark_sent_rtx_req_count; +static GQuark quark_key_unit_requests_count; + /* signals and args */ enum { SIGNAL_REQUEST_PT_MAP, SIGNAL_CLEAR_PT_MAP, + SIGNAL_SEND_BYE, SIGNAL_ON_NEW_SSRC, SIGNAL_ON_SSRC_COLLISION, @@ -293,6 +299,7 @@ struct _GstRtpSessionPrivate GstBufferList *processed_list; gboolean send_rtp_sink_eos; + guint key_unit_requests_count; }; /* callbacks to handle actions from the session manager */ @@ -364,6 +371,7 @@ static gboolean gst_rtp_session_setcaps_send_rtp (GstPad * pad, GstRtpSession * rtpsession, GstCaps * caps); static void gst_rtp_session_clear_pt_map (GstRtpSession * rtpsession); +static void gst_rtp_session_send_bye (GstRtpSession * rtpsession); static GstStructure *gst_rtp_session_create_stats (GstRtpSession * rtpsession); @@ -507,6 +515,12 @@ gst_rtp_session_class_init (GstRtpSessionClass * klass) GObjectClass *gobject_class; GstElementClass *gstelement_class; + quark_rtx_count = g_quark_from_static_string ("rtx-count"); + quark_recv_rtx_req_count = g_quark_from_static_string ("recv-rtx-req-count"); + quark_sent_rtx_req_count = g_quark_from_static_string ("sent-rtx-req-count"); + quark_key_unit_requests_count = + g_quark_from_static_string ("key-unit-requests-count"); + gobject_class = (GObjectClass *) klass; gstelement_class = (GstElementClass *) klass; @@ -537,6 +551,18 @@ gst_rtp_session_class_init (GstRtpSessionClass * klass) G_STRUCT_OFFSET (GstRtpSessionClass, clear_pt_map), NULL, NULL, NULL, G_TYPE_NONE, 0, G_TYPE_NONE); + /** + * GstRtpSession::send-bye: + * @sess: the object which received the signal + * + * Send BYE on all active sources in the session. + */ + gst_rtp_session_signals[SIGNAL_SEND_BYE] = + g_signal_new ("send-bye", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, + G_STRUCT_OFFSET (GstRtpSessionClass, send_bye), + NULL, NULL, g_cclosure_marshal_generic, G_TYPE_NONE, 0, G_TYPE_NONE); + /** * GstRtpSession::on-new-ssrc: * @sess: the object which received the signal @@ -856,6 +882,7 @@ gst_rtp_session_class_init (GstRtpSessionClass * klass) GST_DEBUG_FUNCPTR (gst_rtp_session_release_pad); klass->clear_pt_map = GST_DEBUG_FUNCPTR (gst_rtp_session_clear_pt_map); + klass->send_bye = GST_DEBUG_FUNCPTR (gst_rtp_session_send_bye); /* sink pads */ gst_element_class_add_static_pad_template (gstelement_class, @@ -943,6 +970,7 @@ gst_rtp_session_init (GstRtpSession * rtpsession) rtpsession->priv->recv_rtx_req_count = 0; rtpsession->priv->sent_rtx_req_count = 0; + rtpsession->priv->key_unit_requests_count = 0; rtpsession->priv->ntp_time_source = DEFAULT_NTP_TIME_SOURCE; @@ -1126,13 +1154,16 @@ gst_rtp_session_get_property (GObject * object, guint prop_id, static GstStructure * gst_rtp_session_create_stats (GstRtpSession * rtpsession) { + GstRtpSessionPrivate *priv = rtpsession->priv; GstStructure *s; - g_object_get (rtpsession->priv->session, "stats", &s, NULL); - gst_structure_set (s, "rtx-count", G_TYPE_UINT, - rtpsession->priv->recv_rtx_req_count, "recv-rtx-req-count", G_TYPE_UINT, - rtpsession->priv->recv_rtx_req_count, "sent-rtx-req-count", G_TYPE_UINT, - rtpsession->priv->sent_rtx_req_count, NULL); + g_object_get (priv->session, "stats", &s, NULL); + gst_structure_id_set (s, + quark_rtx_count, G_TYPE_UINT, priv->recv_rtx_req_count, + quark_recv_rtx_req_count, G_TYPE_UINT, priv->recv_rtx_req_count, + quark_sent_rtx_req_count, G_TYPE_UINT, priv->sent_rtx_req_count, + quark_key_unit_requests_count, G_TYPE_UINT, priv->key_unit_requests_count, + NULL); return s; } @@ -1427,6 +1458,15 @@ gst_rtp_session_clear_pt_map (GstRtpSession * rtpsession) GST_RTP_SESSION_UNLOCK (rtpsession); } +static void +gst_rtp_session_send_bye (GstRtpSession * rtpsession) +{ + GstClockTime current_time = gst_clock_get_time (rtpsession->priv->sysclock); + GST_DEBUG_OBJECT (rtpsession, "scheduling BYE message"); + rtp_session_mark_all_bye (rtpsession->priv->session, "End Of Stream"); + rtp_session_schedule_bye (rtpsession->priv->session, current_time); +} + /* called when the session manager has an RTP packet ready to be pushed */ static GstFlowReturn gst_rtp_session_process_rtp (RTPSession * sess, RTPSource * src, @@ -1909,6 +1949,10 @@ gst_rtp_session_event_recv_rtp_src (GstPad * pad, GstObject * parent, if (gst_rtp_session_request_remote_key_unit (rtpsession, ssrc, pt, all_headers, count)) forward = FALSE; + + GST_RTP_SESSION_LOCK (rtpsession); + rtpsession->priv->key_unit_requests_count++; + GST_RTP_SESSION_UNLOCK (rtpsession); } else if (gst_structure_has_name (s, "GstRTPRetransmissionRequest")) { guint seqnum, delay, deadline, max_delay, avg_rtt; @@ -2287,19 +2331,15 @@ gst_rtp_session_event_send_rtp_sink (GstPad * pad, GstObject * parent, break; } case GST_EVENT_EOS:{ - GstClockTime current_time; - /* push downstream FIXME, we are not supposed to leave the session just * because we stop sending. */ ret = gst_pad_push_event (rtpsession->send_rtp_src, event); - current_time = gst_clock_get_time (rtpsession->priv->sysclock); + GST_RTP_SESSION_LOCK (rtpsession); rtpsession->priv->send_rtp_sink_eos = TRUE; GST_RTP_SESSION_UNLOCK (rtpsession); - GST_DEBUG_OBJECT (rtpsession, "scheduling BYE message"); - rtp_session_mark_all_bye (rtpsession->priv->session, "End Of Stream"); - rtp_session_schedule_bye (rtpsession->priv->session, current_time); + gst_rtp_session_send_bye (rtpsession); break; } default:{ @@ -2475,9 +2515,9 @@ gst_rtp_session_chain_send_rtp_common (GstRtpSession * rtpsession, if (priv->send_latency != GST_CLOCK_TIME_NONE) { running_time += priv->send_latency; } else { - GST_WARNING_OBJECT (rtpsession, + GST_INFO_OBJECT (rtpsession, "Can't determine running time for this packet without knowing configured latency"); - running_time = -1; + //running_time = -1; } } } else { diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.h b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.h index 5a3ecfc1c17..d40c9e801d2 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpsession.h @@ -64,6 +64,7 @@ struct _GstRtpSessionClass { /* signals */ GstCaps* (*request_pt_map) (GstRtpSession *sess, guint pt); void (*clear_pt_map) (GstRtpSession *sess); + void (*send_bye) (GstRtpSession *sess); void (*on_new_ssrc) (GstRtpSession *sess, guint32 ssrc); void (*on_ssrc_collision) (GstRtpSession *sess, guint32 ssrc); diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.c index 243449be28f..3ac25327014 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.c @@ -731,9 +731,11 @@ gst_rtp_ssrc_demux_chain (GstPad * pad, GstObject * parent, GstBuffer * buf) create_failed: { gst_buffer_unref (buf); - GST_WARNING_OBJECT (demux, - "Dropping buffer SSRC %08x. " - "Max streams number reached (%u)", ssrc, demux->max_streams); + if ((demux->err_num & 0xff) == 0) + GST_WARNING_OBJECT (demux, + "Dropping buffer SSRC %08x. " + "Max streams number reached (%u)", ssrc, demux->max_streams); + ++demux->err_num; return GST_FLOW_OK; } } diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.h b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.h index e7e347d251d..fecfded69de 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpssrcdemux.h @@ -41,6 +41,7 @@ struct _GstRtpSsrcDemux GRecMutex padlock; GSList *srcpads; guint max_streams; + guint err_num; }; struct _GstRtpSsrcDemuxClass diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/meson.build b/subprojects/gst-plugins-good/gst/rtpmanager/meson.build index 11435d902f0..0a0e6a14a9e 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/meson.build +++ b/subprojects/gst-plugins-good/gst/rtpmanager/meson.build @@ -9,8 +9,10 @@ rtpmanager_sources = [ 'gstrtphdrext-ntp.c', 'gstrtphdrext-repairedstreamid.c', 'gstrtphdrext-streamid.c', + 'gstrtphdrext-roi.c', 'gstrtpmux.c', 'gstrtpptdemux.c', + 'tokenbucket.c', 'gstrtprtxqueue.c', 'gstrtprtxreceive.c', 'gstrtprtxsend.c', diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpjitterbuffer.c index 873fcbe364f..2697f35ab08 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpjitterbuffer.c @@ -530,7 +530,7 @@ update_buffer_level (RTPJitterBuffer * jbuf, gint * percent) */ static GstClockTime calculate_skew (RTPJitterBuffer * jbuf, guint64 ext_rtptime, - GstClockTime gstrtptime, GstClockTime time, gint gap, gboolean is_rtx) + GstClockTime gstrtptime, GstClockTime time, gint gap, gboolean is_redundant) { guint64 send_diff, recv_diff; gint64 delta; @@ -544,7 +544,7 @@ calculate_skew (RTPJitterBuffer * jbuf, guint64 ext_rtptime, /* we don't have an arrival timestamp so we can't do skew detection. we * should still apply a timestamp based on RTP timestamp and base_time */ - if (time == -1 || jbuf->base_time == -1 || is_rtx) + if (time == -1 || jbuf->base_time == -1 || is_redundant) goto no_skew; /* elapsed time at receiver, includes the jitter */ @@ -700,7 +700,7 @@ queue_do_insert (RTPJitterBuffer * jbuf, GList * list, GList * item) GstClockTime rtp_jitter_buffer_calculate_pts (RTPJitterBuffer * jbuf, GstClockTime dts, gboolean estimated_dts, guint32 rtptime, GstClockTime base_time, - gint gap, gboolean is_rtx, GstClockTime * p_ntp_time) + gint gap, gboolean is_redundant, GstClockTime * p_ntp_time) { guint64 ext_rtptime; GstClockTime gstrtptime, pts; @@ -723,13 +723,13 @@ rtp_jitter_buffer_calculate_pts (RTPJitterBuffer * jbuf, GstClockTime dts, ext_rtptime = gst_rtp_buffer_ext_timestamp (&ext_rtptime, rtptime); if (ext_rtptime > jbuf->last_rtptime + 3 * jbuf->clock_rate || ext_rtptime + 3 * jbuf->clock_rate < jbuf->last_rtptime) { - if (!is_rtx) { + if (!is_redundant) { /* reset even if we don't have valid incoming time; * still better than producing possibly very bogus output timestamp */ GST_WARNING ("rtp delta too big, reset skew"); rtp_jitter_buffer_reset_skew (jbuf); } else { - GST_WARNING ("rtp delta too big: ignore rtx packet"); + GST_WARNING ("rtp delta too big: ignore redundant packet"); media_clock = NULL; pipeline_clock = NULL; pts = GST_CLOCK_TIME_NONE; @@ -760,14 +760,14 @@ rtp_jitter_buffer_calculate_pts (RTPJitterBuffer * jbuf, GstClockTime dts, if (G_LIKELY (jbuf->base_rtptime != -1)) { /* check elapsed time in RTP units */ if (gstrtptime < jbuf->base_rtptime) { - if (!is_rtx) { + if (!is_redundant) { /* elapsed time at sender, timestamps can go backwards and thus be * smaller than our base time, schedule to take a new base time in * that case. */ GST_WARNING ("backward timestamps at server, schedule resync"); jbuf->need_resync = TRUE; } else { - GST_WARNING ("backward timestamps: ignore rtx packet"); + GST_DEBUG ("backward timestamps: ignore redundant packet"); pts = GST_CLOCK_TIME_NONE; goto done; } @@ -800,8 +800,8 @@ rtp_jitter_buffer_calculate_pts (RTPJitterBuffer * jbuf, GstClockTime dts, /* need resync, lock on to time and gstrtptime if we can, otherwise we * do with the previous values */ if (G_UNLIKELY (jbuf->need_resync && dts != -1)) { - if (is_rtx) { - GST_DEBUG ("not resyncing on rtx packet, discard"); + if (is_redundant) { + GST_DEBUG ("not resyncing on redundant packet, discard"); pts = GST_CLOCK_TIME_NONE; goto done; } @@ -1004,7 +1004,8 @@ rtp_jitter_buffer_calculate_pts (RTPJitterBuffer * jbuf, GstClockTime dts, /* do skew calculation by measuring the difference between rtptime and the * receive dts, this function will return the skew corrected rtptime. */ - pts = calculate_skew (jbuf, ext_rtptime, gstrtptime, dts, gap, is_rtx); + pts = + calculate_skew (jbuf, ext_rtptime, gstrtptime, dts, gap, is_redundant); } /* check if timestamps are not going backwards, we can only check this if we @@ -1027,12 +1028,18 @@ rtp_jitter_buffer_calculate_pts (RTPJitterBuffer * jbuf, GstClockTime dts, } if (gap == 0 && dts != -1 && pts + jbuf->delay < dts) { + if (is_redundant) { + GST_DEBUG ("avoiding reset on redundant packet, discard"); + pts = GST_CLOCK_TIME_NONE; + goto done; + } + /* if we are going to produce a timestamp that is later than the input * timestamp, we need to reset the jitterbuffer. Likely the server paused * temporarily */ GST_DEBUG ("out %" GST_TIME_FORMAT " + %" G_GUINT64_FORMAT " < time %" - GST_TIME_FORMAT ", reset jitterbuffer and discard", GST_TIME_ARGS (pts), - jbuf->delay, GST_TIME_ARGS (dts)); + GST_TIME_FORMAT ", reset jitterbuffer and discard", + GST_TIME_ARGS (pts), jbuf->delay, GST_TIME_ARGS (dts)); rtp_jitter_buffer_reset_skew (jbuf); rtp_jitter_buffer_resync (jbuf, dts, gstrtptime, ext_rtptime, TRUE); pts = dts; diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c index 70c08774304..6dbba5de5a6 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.c @@ -39,15 +39,18 @@ GST_DEBUG_CATEGORY (rtp_session_debug); enum { SIGNAL_GET_SOURCE_BY_SSRC, + SIGNAL_GET_TWCC_WINDOWED_STATS, SIGNAL_ON_NEW_SSRC, SIGNAL_ON_SSRC_COLLISION, SIGNAL_ON_SSRC_VALIDATED, SIGNAL_ON_SSRC_ACTIVE, SIGNAL_ON_SSRC_SDES, + SIGNAL_ON_SSRC_PROFILE_SPECIFIC_EXT, SIGNAL_ON_BYE_SSRC, SIGNAL_ON_BYE_TIMEOUT, SIGNAL_ON_TIMEOUT, SIGNAL_ON_SENDER_TIMEOUT, + SIGNAL_ON_CREATING_SR_RR, SIGNAL_ON_SENDING_RTCP, SIGNAL_ON_APP_RTCP, SIGNAL_ON_FEEDBACK_RTCP, @@ -57,6 +60,7 @@ enum SIGNAL_ON_NEW_SENDER_SSRC, SIGNAL_ON_SENDER_SSRC_ACTIVE, SIGNAL_ON_SENDING_NACKS, + SIGNAL_NACK_PROBE, LAST_SIGNAL }; @@ -76,6 +80,7 @@ enum #define DEFAULT_PROBATION RTP_DEFAULT_PROBATION #define DEFAULT_MAX_DROPOUT_TIME 60000 #define DEFAULT_MAX_MISORDER_TIME 2000 +#define DEFAULT_SSRC_COLLISION_DETECTION TRUE #define DEFAULT_RTP_PROFILE GST_RTP_PROFILE_AVP #define DEFAULT_RTCP_REDUCED_SIZE FALSE #define DEFAULT_RTCP_DISABLE_SR_TIMESTAMP FALSE @@ -83,6 +88,7 @@ enum #define DEFAULT_TWCC_FEEDBACK_INTERVAL GST_CLOCK_TIME_NONE #define DEFAULT_UPDATE_NTP64_HEADER_EXT TRUE #define DEFAULT_TIMEOUT_INACTIVE_SOURCES TRUE +#define DEFAULT_STATS_NOTIFY_MIN_INTERVAL 0 enum { @@ -105,13 +111,16 @@ enum PROP_PROBATION, PROP_MAX_DROPOUT_TIME, PROP_MAX_MISORDER_TIME, + PROP_SSRC_COLLISION_DETECTION, PROP_STATS, + PROP_STATS_NOTIFY_MIN_INTERVAL, PROP_RTP_PROFILE, PROP_RTCP_REDUCED_SIZE, PROP_RTCP_DISABLE_SR_TIMESTAMP, PROP_TWCC_FEEDBACK_INTERVAL, PROP_UPDATE_NTP64_HEADER_EXT, PROP_TIMEOUT_INACTIVE_SOURCES, + PROP_RTX_SSRC_MAP, PROP_LAST, }; @@ -133,13 +142,25 @@ static void rtp_session_set_property (GObject * object, guint prop_id, static void rtp_session_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); +static GstStructure *rtp_session_get_twcc_windowed_stats (RTPSession * sess, + GstClockTime stats_window_size, GstClockTime stats_window_delay); + static gboolean rtp_session_send_rtcp (RTPSession * sess, GstClockTime max_delay); static gboolean rtp_session_send_rtcp_with_deadline (RTPSession * sess, GstClockTime deadline); +static void rtp_session_nack_probe (RTPSession * sess, + guint ssrc, guint pct, GstClockTime duration); static guint rtp_session_signals[LAST_SIGNAL] = { 0 }; +static GQuark quark_application_x_rtp_session_stats; +static GQuark quark_recv_nack_count; +static GQuark quark_rtx_drop_count; +static GQuark quark_sent_nack_count; +static GQuark quark_source_stats; + + G_DEFINE_TYPE (RTPSession, rtp_session, G_TYPE_OBJECT); static guint32 rtp_session_create_new_ssrc (RTPSession * sess); @@ -167,6 +188,13 @@ rtp_session_class_init (RTPSessionClass * klass) { GObjectClass *gobject_class; + quark_application_x_rtp_session_stats = + g_quark_from_static_string ("application/x-rtp-session-stats"); + quark_recv_nack_count = g_quark_from_static_string ("recv-nack-count"); + quark_rtx_drop_count = g_quark_from_static_string ("rtx-drop-count"); + quark_sent_nack_count = g_quark_from_static_string ("sent-nack-count"); + quark_source_stats = g_quark_from_static_string ("source-stats"); + gobject_class = (GObjectClass *) klass; gobject_class->finalize = rtp_session_finalize; @@ -186,6 +214,33 @@ rtp_session_class_init (RTPSessionClass * klass) get_source_by_ssrc), NULL, NULL, NULL, RTP_TYPE_SOURCE, 1, G_TYPE_UINT); + /** + * RTPSession::get_twcc_windowed_stats: + * @session: the object which received the signal + * @stats_window_size: The size (in nanoseconds) of the stats window + * @stats_window_delay: The delay (in nanoseconds) from the current time + * until the start of the stats window. + * + * While the size of the stats window is the time to consider stats over, + * the delay is in effect a "jitterbuffer latency" for the send-side, allowing + * effects such as reordering and recovery to affect the stats, potentially + * reporting a much prettier picture of the network conditions. + * + * As an example, if the current time was 50ms, + * this would be a window size of 20ms, and a window + * delay of 10ms: + * + * timeline (ms): 0 10 20 30 40 50 + * window: [------------] + * + * Since: 1.24 + */ + rtp_session_signals[SIGNAL_GET_TWCC_WINDOWED_STATS] = + g_signal_new ("get-twcc-windowed-stats", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, G_STRUCT_OFFSET (RTPSessionClass, + get_twcc_windowed_stats), NULL, NULL, NULL, + GST_TYPE_STRUCTURE, 2, GST_TYPE_CLOCK_TIME, GST_TYPE_CLOCK_TIME); + /** * RTPSession::on-new-ssrc: * @session: the object which received the signal @@ -241,6 +296,24 @@ rtp_session_class_init (RTPSessionClass * klass) g_signal_new ("on-ssrc-sdes", G_TYPE_FROM_CLASS (klass), G_SIGNAL_RUN_LAST, G_STRUCT_OFFSET (RTPSessionClass, on_ssrc_sdes), NULL, NULL, NULL, G_TYPE_NONE, 1, RTP_TYPE_SOURCE); + /** + * RTPSession::on-ssrc-profile-specific-ext: + * @session: the object which received the signal + * @src: the RTPSource + * @type: Type of RTCP packet, will be %GST_RTCP_TYPE_SR or + * %GST_RTCP_TYPE_RR + * @pse: a #GstBuffer with the profile-specific extensions data from the + * SR or RR packet + * @rb_count: report block count + * + * Notify that a RTCP profile-specific extension section has been received + */ + rtp_session_signals[SIGNAL_ON_SSRC_PROFILE_SPECIFIC_EXT] = + g_signal_new ("on-ssrc-profile-specific-ext", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST, G_STRUCT_OFFSET (RTPSessionClass, on_ssrc_pse), + NULL, NULL, g_cclosure_marshal_generic, + G_TYPE_NONE, 4, RTP_TYPE_SOURCE, G_TYPE_UINT, GST_TYPE_BUFFER, + G_TYPE_UINT); /** * RTPSession::on-bye-ssrc: * @session: the object which received the signal @@ -286,6 +359,21 @@ rtp_session_class_init (RTPSessionClass * klass) G_SIGNAL_RUN_LAST, G_STRUCT_OFFSET (RTPSessionClass, on_sender_timeout), NULL, NULL, NULL, G_TYPE_NONE, 1, RTP_TYPE_SOURCE); + /** + * RTPSession::on-creating-sr-rr + * @session: the object which received the signal + * @src: the RTPSource + * @packet: the #GstRTCPPacket which was just created. + * + * This signal is emitted while creating RTCP SR or RR packet, but before + * sending it. It can be used to add profile-specific extension. + */ + rtp_session_signals[SIGNAL_ON_CREATING_SR_RR] = + g_signal_new ("on-creating-sr-rr", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST, G_STRUCT_OFFSET (RTPSessionClass, on_creating_sr_rr), + NULL, NULL, g_cclosure_marshal_generic, G_TYPE_NONE, 2, + RTP_TYPE_SOURCE, G_TYPE_POINTER); + /** * RTPSession::on-sending-rtcp * @session: the object which received the signal @@ -424,6 +512,23 @@ rtp_session_class_init (RTPSessionClass * klass) on_sender_ssrc_active), NULL, NULL, NULL, G_TYPE_NONE, 1, RTP_TYPE_SOURCE); + /** + * RTPSession::nack-probe: + * @session: the object which received the signal + * @pct: The percentage of the received packets to send NACK for + * @duration: the duration to send NACK for + * @ssrc: the SSRC to send NACK for. + * + * Requests that the #RTPSession sends NACK on a certain percent of the + * received RTP packets. + */ + rtp_session_signals[SIGNAL_NACK_PROBE] = + g_signal_new ("nack-probe", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION, + G_STRUCT_OFFSET (RTPSessionClass, nack_probe), NULL, NULL, + g_cclosure_marshal_generic, + G_TYPE_NONE, 3, G_TYPE_UINT, G_TYPE_UINT, G_TYPE_UINT64); + /** * RTPSession::on-sending-nack * @session: the object which received the signal @@ -587,6 +692,13 @@ rtp_session_class_init (RTPSessionClass * klass) 0, G_MAXUINT, DEFAULT_MAX_MISORDER_TIME, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + properties[PROP_SSRC_COLLISION_DETECTION] = + g_param_spec_boolean ("ssrc-collision-detection", + "SSRC collision detection", + "Enable/Disable SSRC collision detection for remote sources", + DEFAULT_SSRC_COLLISION_DETECTION, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + /** * RTPSession:stats: * @@ -607,6 +719,13 @@ rtp_session_class_init (RTPSessionClass * klass) "Various statistics", GST_TYPE_STRUCTURE, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS); + properties[PROP_STATS_NOTIFY_MIN_INTERVAL] = + g_param_spec_uint ("stats-notify-min-interval", + "Stats notify minimum interval", + "Minimum interval between emitting notify signal for stats (in ms)", + 0, G_MAXUINT, DEFAULT_STATS_NOTIFY_MIN_INTERVAL, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + properties[PROP_RTP_PROFILE] = g_param_spec_enum ("rtp-profile", "RTP Profile", "RTP profile to use for this session", GST_TYPE_RTP_PROFILE, @@ -677,11 +796,26 @@ rtp_session_class_init (RTPSessionClass * klass) DEFAULT_TIMEOUT_INACTIVE_SOURCES, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + /** + * RTPSession:rtx-ssrc-map: + * + * Mapping from SSRC to RTX ssrcs. + * + * Since: 1.24 + */ + properties[PROP_RTX_SSRC_MAP] = + g_param_spec_boxed ("rtx-ssrc-map", "RTX SSRC Map", + "Map of SSRCs to their retransmission SSRCs", + GST_TYPE_STRUCTURE, G_PARAM_WRITABLE | G_PARAM_STATIC_STRINGS); + g_object_class_install_properties (gobject_class, PROP_LAST, properties); + klass->get_twcc_windowed_stats = + GST_DEBUG_FUNCPTR (rtp_session_get_twcc_windowed_stats); klass->get_source_by_ssrc = GST_DEBUG_FUNCPTR (rtp_session_get_source_by_ssrc); klass->send_rtcp = GST_DEBUG_FUNCPTR (rtp_session_send_rtcp); + klass->nack_probe = GST_DEBUG_FUNCPTR (rtp_session_nack_probe); GST_DEBUG_CATEGORY_INIT (rtp_session_debug, "rtpsession", 0, "RTP Session"); } @@ -729,6 +863,7 @@ rtp_session_init (RTPSession * sess) sess->max_dropout_time = DEFAULT_MAX_DROPOUT_TIME; sess->max_misorder_time = DEFAULT_MAX_MISORDER_TIME; sess->favor_new = DEFAULT_FAVOR_NEW; + sess->ssrc_collision_detection = DEFAULT_SSRC_COLLISION_DETECTION; /* some default SDES entries */ sess->sdes = gst_structure_new_empty ("application/x-rtp-source-sdes"); @@ -758,6 +893,7 @@ rtp_session_init (RTPSession * sess) sess->last_rtcp_interval = GST_CLOCK_TIME_NONE; sess->next_early_rtcp_time = GST_CLOCK_TIME_NONE; + sess->next_twcc_rtcp_time = GST_CLOCK_TIME_NONE; sess->rtcp_feedback_retention_window = DEFAULT_RTCP_FEEDBACK_RETENTION_WINDOW; sess->rtcp_immediate_feedback_threshold = DEFAULT_RTCP_IMMEDIATE_FEEDBACK_THRESHOLD; @@ -765,10 +901,13 @@ rtp_session_init (RTPSession * sess) sess->reduced_size_rtcp = DEFAULT_RTCP_REDUCED_SIZE; sess->timestamp_sender_reports = !DEFAULT_RTCP_DISABLE_SR_TIMESTAMP; + sess->last_stats_notify_time = GST_CLOCK_TIME_NONE; + sess->stats_notify_min_interval_ms = DEFAULT_STATS_NOTIFY_MIN_INTERVAL; + sess->is_doing_ptp = TRUE; sess->twcc = rtp_twcc_manager_new (sess->mtu); - sess->twcc_stats = rtp_twcc_stats_new (); + sess->rtx_ssrc_to_ssrc = g_hash_table_new (NULL, NULL); } static void @@ -791,7 +930,9 @@ rtp_session_finalize (GObject * object) g_hash_table_destroy (sess->ssrcs[i]); g_object_unref (sess->twcc); - rtp_twcc_stats_free (sess->twcc_stats); + if (sess->rtx_ssrc_map) + gst_structure_free (sess->rtx_ssrc_map); + g_hash_table_destroy (sess->rtx_ssrc_to_ssrc); g_mutex_clear (&sess->lock); @@ -851,10 +992,10 @@ rtp_session_create_stats (RTPSession * sess) guint size; RTP_SESSION_LOCK (sess); - s = gst_structure_new ("application/x-rtp-session-stats", - "rtx-drop-count", G_TYPE_UINT, sess->stats.nacks_dropped, - "sent-nack-count", G_TYPE_UINT, sess->stats.nacks_sent, - "recv-nack-count", G_TYPE_UINT, sess->stats.nacks_received, NULL); + s = gst_structure_new_id (quark_application_x_rtp_session_stats, + quark_rtx_drop_count, G_TYPE_UINT, sess->stats.nacks_dropped, + quark_sent_nack_count, G_TYPE_UINT, sess->stats.nacks_sent, + quark_recv_nack_count, G_TYPE_UINT, sess->stats.nacks_received, NULL); size = g_hash_table_size (sess->ssrcs[sess->mask_idx]); source_stats = g_value_array_new (size); @@ -864,11 +1005,28 @@ rtp_session_create_stats (RTPSession * sess) g_value_init (&source_stats_v, G_TYPE_VALUE_ARRAY); g_value_take_boxed (&source_stats_v, source_stats); - gst_structure_take_value (s, "source-stats", &source_stats_v); + gst_structure_id_take_value (s, quark_source_stats, &source_stats_v); return s; } +static gboolean +structure_to_hash_table_reverse (GQuark field_id, const GValue * value, + gpointer hash) +{ + const gchar *field_str; + guint field_uint; + guint value_uint; + + field_str = g_quark_to_string (field_id); + field_uint = atoi (field_str); + value_uint = g_value_get_uint (value); + g_hash_table_insert ((GHashTable *) hash, GUINT_TO_POINTER (value_uint), + GUINT_TO_POINTER (field_uint)); + + return TRUE; +} + static void rtp_session_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) @@ -946,6 +1104,12 @@ rtp_session_set_property (GObject * object, guint prop_id, case PROP_MAX_MISORDER_TIME: sess->max_misorder_time = g_value_get_uint (value); break; + case PROP_STATS_NOTIFY_MIN_INTERVAL: + sess->stats_notify_min_interval_ms = g_value_get_uint (value); + break; + case PROP_SSRC_COLLISION_DETECTION: + sess->ssrc_collision_detection = g_value_get_boolean (value); + break; case PROP_RTP_PROFILE: sess->rtp_profile = g_value_get_enum (value); /* trigger reconsideration */ @@ -971,6 +1135,17 @@ rtp_session_set_property (GObject * object, guint prop_id, case PROP_TIMEOUT_INACTIVE_SOURCES: sess->timeout_inactive_sources = g_value_get_boolean (value); break; + case PROP_RTX_SSRC_MAP: + RTP_SESSION_LOCK (sess); + if (sess->rtx_ssrc_map) + gst_structure_free (sess->rtx_ssrc_map); + sess->rtx_ssrc_map = g_value_dup_boxed (value); + g_hash_table_remove_all (sess->rtx_ssrc_to_ssrc); + gst_structure_foreach (sess->rtx_ssrc_map, + structure_to_hash_table_reverse, sess->rtx_ssrc_to_ssrc); + RTP_SESSION_UNLOCK (sess); + break; + default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1041,9 +1216,15 @@ rtp_session_get_property (GObject * object, guint prop_id, case PROP_MAX_MISORDER_TIME: g_value_set_uint (value, sess->max_misorder_time); break; + case PROP_SSRC_COLLISION_DETECTION: + g_value_set_boolean (value, sess->ssrc_collision_detection); + break; case PROP_STATS: g_value_take_boxed (value, rtp_session_create_stats (sess)); break; + case PROP_STATS_NOTIFY_MIN_INTERVAL: + g_value_set_uint (value, sess->stats_notify_min_interval_ms); + break; case PROP_RTP_PROFILE: g_value_set_enum (value, sess->rtp_profile); break; @@ -1229,6 +1410,7 @@ rtp_session_reset (RTPSession * sess) sess->last_rtcp_send_time = GST_CLOCK_TIME_NONE; sess->last_rtcp_interval = GST_CLOCK_TIME_NONE; sess->next_early_rtcp_time = GST_CLOCK_TIME_NONE; + sess->next_twcc_rtcp_time = GST_CLOCK_TIME_NONE; sess->scheduled_bye = FALSE; /* reset session stats */ @@ -1245,6 +1427,29 @@ rtp_session_reset (RTPSession * sess) RTP_SESSION_UNLOCK (sess); } +/* must be called with RTP_SESSION_LOCK held */ +static GstCaps * +rtp_session_get_caps_unlocked (RTPSession * sess, guint8 pt) +{ + GstCaps *ret = NULL; + + RTP_SESSION_UNLOCK (sess); + + if (sess->callbacks.caps) + ret = sess->callbacks.caps (sess, pt, sess->caps_user_data); + + RTP_SESSION_LOCK (sess); + + return ret; +} + +static GstCaps * +rtp_session_twcc_caps_cb (guint8 payload, gpointer user_data) +{ + RTPSession *sess = user_data; + return rtp_session_get_caps_unlocked (sess, payload); +} + /** * rtp_session_set_callbacks: * @sess: an #RTPSession @@ -1278,6 +1483,7 @@ rtp_session_set_callbacks (RTPSession * sess, RTPSessionCallbacks * callbacks, if (callbacks->caps) { sess->callbacks.caps = callbacks->caps; sess->caps_user_data = user_data; + rtp_twcc_manager_set_callback (sess->twcc, rtp_session_twcc_caps_cb, sess); } if (callbacks->reconsider) { sess->callbacks.reconsider = callbacks->reconsider; @@ -1605,12 +1811,7 @@ source_caps (RTPSource * source, guint8 pt, RTPSession * session) { GstCaps *result = NULL; - RTP_SESSION_UNLOCK (session); - - if (session->callbacks.caps) - result = session->callbacks.caps (session, pt, session->caps_user_data); - - RTP_SESSION_LOCK (session); + result = rtp_session_get_caps_unlocked (session, pt); GST_DEBUG ("got caps %" GST_PTR_FORMAT " for pt %d", result, pt); @@ -1703,7 +1904,7 @@ check_collision (RTPSession * sess, RTPSource * source, from = source->rtcp_from; } - if (from) { + if (from && sess->ssrc_collision_detection) { if (__g_socket_address_equal (from, pinfo->address)) { /* Address is the same */ return FALSE; @@ -1715,11 +1916,11 @@ check_collision (RTPSession * sess, RTPSource * source, gchar *buf1; buf1 = __g_socket_address_to_string (pinfo->address); - GST_LOG ("Known conflict on %x for %s, dropping packet", ssrc, - buf1); + GST_LOG ("Known conflict on %x for %s, %s packet", ssrc, + buf1, sess->ssrc_collision_detection ? "dropping" : "ignoring"); g_free (buf1); - return TRUE; + return sess->ssrc_collision_detection; } else { gchar *buf1, *buf2; @@ -1747,15 +1948,17 @@ check_collision (RTPSession * sess, RTPSource * source, } } else { /* Don't need to save old addresses, we ignore new sources */ - return TRUE; + return sess->ssrc_collision_detection; /* if enabled */ } } } else { /* We don't already have a from address for RTP, just set it */ - if (rtp) - rtp_source_set_rtp_from (source, pinfo->address); - else - rtp_source_set_rtcp_from (source, pinfo->address); + if ((from == NULL) || !__g_socket_address_equal (from, pinfo->address)) { + if (rtp) + rtp_source_set_rtp_from (source, pinfo->address); + else + rtp_source_set_rtcp_from (source, pinfo->address); + } return FALSE; } @@ -1917,7 +2120,7 @@ obtain_source (RTPSession * sess, guint32 ssrc, gboolean * created, /* for RTP packets we need to set the source in probation. Receiving RTCP * packets of an SSRC, on the other hand, is a strong indication that we * are dealing with a valid source. */ - g_object_set (source, "probation", rtp ? sess->probation : 0, + g_object_set (source, "probation", rtp ? sess->probation : RTP_NO_PROBATION, "max-dropout-time", sess->max_dropout_time, "max-misorder-time", sess->max_misorder_time, NULL); @@ -1943,12 +2146,15 @@ obtain_source (RTPSession * sess, guint32 ssrc, gboolean * created, /* Receiving RTCP packets of an SSRC is a strong indication that we * are dealing with a valid source. */ if (!rtp) - g_object_set (source, "probation", 0, NULL); + g_object_set (source, "probation", RTP_NO_PROBATION, NULL); } /* update last activity */ source->last_activity = pinfo->current_time; - if (rtp) + if (rtp) { source->last_rtp_activity = pinfo->current_time; + if (source->first_rtp_activity == GST_CLOCK_TIME_NONE) + source->first_rtp_activity = pinfo->current_time; + } g_object_ref (source); return source; @@ -1967,7 +2173,7 @@ obtain_internal_source (RTPSession * sess, guint32 ssrc, gboolean * created, /* make new internal Source and insert */ source = rtp_source_new (ssrc); - GST_DEBUG ("creating new internal source %08x %p", ssrc, source); + GST_INFO ("creating new internal source %08x %p", ssrc, source); source->validated = TRUE; source->internal = TRUE; @@ -1985,6 +2191,8 @@ obtain_internal_source (RTPSession * sess, guint32 ssrc, gboolean * created, if (current_time != GST_CLOCK_TIME_NONE) { source->last_activity = current_time; source->last_rtp_activity = current_time; + if (source->first_rtp_activity == GST_CLOCK_TIME_NONE) + source->first_rtp_activity = current_time; } g_object_ref (source); @@ -2117,6 +2325,22 @@ rtp_session_get_source_by_ssrc (RTPSession * sess, guint32 ssrc) return result; } +static GstStructure * +rtp_session_get_twcc_windowed_stats (RTPSession * sess, + GstClockTime stats_window_size, GstClockTime stats_window_delay) +{ + GstStructure *ret; + + g_return_val_if_fail (RTP_IS_SESSION (sess), NULL); + + RTP_SESSION_LOCK (sess); + ret = rtp_twcc_manager_get_windowed_stats (sess->twcc, + stats_window_size, stats_window_delay); + RTP_SESSION_UNLOCK (sess); + + return ret; +} + /* should be called with the SESSION lock */ static guint32 rtp_session_create_new_ssrc (RTPSession * sess) @@ -2166,6 +2390,14 @@ update_packet (GstBuffer ** buffer, guint idx, RTPPacketInfo * pinfo) /* RTP header extensions */ pinfo->header_ext = gst_rtp_buffer_get_extension_bytes (&rtp, &pinfo->header_ext_bit_pattern); + + /* if RTX, store the original seqnum (OSN) and SSRC */ + if (GST_BUFFER_FLAG_IS_SET (*buffer, GST_RTP_BUFFER_FLAG_RETRANSMISSION)) { + guint8 *payload = gst_rtp_buffer_get_payload (&rtp); + if (payload) { + pinfo->rtx_osn = GST_READ_UINT16_BE (payload); + } + } } if (pinfo->ntp64_ext_id != 0 && pinfo->send && !pinfo->have_ntp64_ext) { @@ -2236,6 +2468,8 @@ update_packet_info (RTPSession * sess, RTPPacketInfo * pinfo, pinfo->marker = FALSE; pinfo->ntp64_ext_id = send ? sess->send_ntp64_ext_id : 0; pinfo->have_ntp64_ext = FALSE; + pinfo->rtx_osn = -1; + pinfo->rtx_ssrc = 0; if (is_list) { GstBufferList *list = GST_BUFFER_LIST_CAST (data); @@ -2249,6 +2483,11 @@ update_packet_info (RTPSession * sess, RTPPacketInfo * pinfo, pinfo->arrival_time = GST_BUFFER_DTS (buffer); } + if (pinfo->rtx_osn != -1) + pinfo->rtx_ssrc = + GPOINTER_TO_UINT (g_hash_table_lookup (sess->rtx_ssrc_to_ssrc, + GUINT_TO_POINTER (pinfo->ssrc))); + return res; } @@ -2295,7 +2534,7 @@ process_twcc_packet (RTPSession * sess, RTPPacketInfo * pinfo) /* TODO: find a better rational for this number, and possibly tune it based on factors like framerate / bandwidth etc */ - if (!rtp_session_send_rtcp (sess, 100 * GST_MSECOND)) { + if (!rtp_session_send_rtcp (sess, 0)) { GST_INFO ("Could not schedule TWCC straight away"); } RTP_SESSION_LOCK (sess); @@ -2328,6 +2567,34 @@ source_update_sender (RTPSession * sess, RTPSource * source, return TRUE; } +static void +rtp_session_process_nack_probe (RTPSession * sess, RTPSource * source, + guint16 seqnum, GstClockTime current_time) +{ + if (source->ssrc != sess->nack_probe_ssrc) + return; + + /* we have a valid duration, set the deadline */ + if (GST_CLOCK_TIME_IS_VALID (sess->nack_probe_duration) && + !GST_CLOCK_TIME_IS_VALID (sess->nack_probe_deadline)) { + sess->nack_probe_deadline = current_time + sess->nack_probe_duration; + sess->nack_probe_duration = GST_CLOCK_TIME_NONE; + } + + /* we have a valid deadline, send NACK or timeout */ + if (GST_CLOCK_TIME_IS_VALID (sess->nack_probe_deadline)) { + if (current_time <= sess->nack_probe_deadline) { + guint pct = (guint) (g_random_double () * 100.0); + if ((pct + sess->nack_probe_pct) > 100) { + GST_DEBUG ("Register nack for ssrc: %u #%u", source->ssrc, seqnum); + rtp_source_register_nack (source, seqnum, sess->nack_probe_deadline); + } + } else { + sess->nack_probe_deadline = GST_CLOCK_TIME_NONE; + } + } +} + /** * rtp_session_process_rtp: * @sess: and #RTPSession @@ -2383,6 +2650,8 @@ rtp_session_process_rtp (RTPSession * sess, GstBuffer * buffer, result = rtp_source_process_rtp (source, &pinfo); process_twcc_packet (sess, &pinfo); + rtp_session_process_nack_probe (sess, source, pinfo.seqnum, current_time); + /* source became active */ if (source_update_active (sess, source, prevactive)) on_ssrc_validated (sess, source); @@ -2464,13 +2733,48 @@ rtp_session_process_rb (RTPSession * sess, RTPSource * source, * the sender of the RTCP message. We could also compare our stats against * the other sender to see if we are better or worse. */ /* FIXME, need to keep track who the RB block is from */ - rtp_source_process_rb (source, ssrc, pinfo->ntpnstime, fractionlost, + rtp_source_process_rb (src, ssrc, pinfo->ntpnstime, fractionlost, packetslost, exthighestseq, jitter, lsr, dlsr); } } on_ssrc_active (sess, source); } +static void +rtp_session_process_pse (RTPSession * sess, RTPSource * source, + GstRTCPPacket * packet, RTPPacketInfo * pinfo) +{ + guint pse_length, type, rb_count; + guint8 *pse_data; + GstBuffer *pse_buffer; + + if (!gst_rtcp_packet_get_profile_specific_ext (packet, &pse_data, + &pse_length)) + return; + + if (!g_signal_has_handler_pending (sess, + rtp_session_signals[SIGNAL_ON_SSRC_PROFILE_SPECIFIC_EXT], 0, TRUE)) + return; + + GST_DEBUG ("got SR/RR packet with PSE: SSRC %08x, PSE length %u", + source->ssrc, pse_length); + + pse_buffer = gst_buffer_copy_region (packet->rtcp->buffer, + GST_BUFFER_COPY_MEMORY, pse_data - packet->rtcp->map.data, pse_length); + GST_BUFFER_TIMESTAMP (pse_buffer) = pinfo->running_time; + type = gst_rtcp_packet_get_type (packet); + rb_count = gst_rtcp_packet_get_rb_count (packet); + + g_object_ref (source); + RTP_SESSION_UNLOCK (sess); + g_signal_emit (sess, rtp_session_signals[SIGNAL_ON_SSRC_PROFILE_SPECIFIC_EXT], + 0, source, type, pse_buffer, rb_count); + RTP_SESSION_LOCK (sess); + g_object_unref (source); + + gst_buffer_unref (pse_buffer); +} + /* A Sender report contains statistics about how the sender is doing. This * includes timing informataion such as the relation between RTP and NTP * timestamps and the number of packets/bytes it sent to us. @@ -2521,6 +2825,7 @@ rtp_session_process_sr (RTPSession * sess, GstRTCPPacket * packet, on_new_ssrc (sess, source); rtp_session_process_rb (sess, source, packet, pinfo); + rtp_session_process_pse (sess, source, packet, pinfo); out: g_object_unref (source); @@ -2556,6 +2861,7 @@ rtp_session_process_rr (RTPSession * sess, GstRTCPPacket * packet, on_new_ssrc (sess, source); rtp_session_process_rb (sess, source, packet, pinfo); + rtp_session_process_pse (sess, source, packet, pinfo); out: g_object_unref (source); @@ -2976,26 +3282,23 @@ rtp_session_process_sr_req (RTPSession * sess, guint32 sender_ssrc, static void rtp_session_process_twcc (RTPSession * sess, guint32 sender_ssrc, - guint32 media_ssrc, guint8 * fci_data, guint fci_length) + guint32 media_ssrc, guint8 * fci_data, guint fci_length, + GstClockTime current_time) { - GArray *twcc_packets; GstStructure *twcc_packets_s; GstStructure *twcc_stats_s; - twcc_packets = rtp_twcc_manager_parse_fci (sess->twcc, - fci_data, fci_length * sizeof (guint32)); - if (twcc_packets == NULL) + twcc_packets_s = rtp_twcc_manager_parse_fci (sess->twcc, + fci_data, fci_length * sizeof (guint32), current_time); + if (twcc_packets_s == NULL) return; - twcc_packets_s = rtp_twcc_stats_get_packets_structure (twcc_packets); - twcc_stats_s = - rtp_twcc_stats_process_packets (sess->twcc_stats, twcc_packets); + twcc_stats_s = rtp_twcc_manager_get_windowed_stats (sess->twcc, + 300 * GST_MSECOND, 200 * GST_MSECOND); GST_DEBUG_OBJECT (sess, "Parsed TWCC: %" GST_PTR_FORMAT, twcc_packets_s); GST_INFO_OBJECT (sess, "Current TWCC stats %" GST_PTR_FORMAT, twcc_stats_s); - g_array_unref (twcc_packets); - RTP_SESSION_UNLOCK (sess); if (sess->callbacks.notify_twcc) sess->callbacks.notify_twcc (sess, twcc_packets_s, twcc_stats_s, @@ -3099,7 +3402,7 @@ rtp_session_process_feedback (RTPSession * sess, GstRTCPPacket * packet, break; case GST_RTCP_RTPFB_TYPE_TWCC: rtp_session_process_twcc (sess, sender_ssrc, media_ssrc, - fci_data, fci_length); + fci_data, fci_length, current_time); break; default: break; @@ -3130,7 +3433,7 @@ rtp_session_process_rtcp (RTPSession * sess, GstBuffer * buffer, GstClockTime current_time, GstClockTime running_time, guint64 ntpnstime) { GstRTCPPacket packet; - gboolean more, is_bye = FALSE, do_sync = FALSE, has_report = FALSE; + gboolean more, is_bye = FALSE, do_sync = FALSE; RTPPacketInfo pinfo = { 0, }; GstFlowReturn result = GST_FLOW_OK; GstRTCPBuffer rtcp = { NULL, }; @@ -3161,11 +3464,9 @@ rtp_session_process_rtcp (RTPSession * sess, GstBuffer * buffer, switch (type) { case GST_RTCP_TYPE_SR: - has_report = TRUE; rtp_session_process_sr (sess, &packet, &pinfo, &do_sync); break; case GST_RTCP_TYPE_RR: - has_report = TRUE; rtp_session_process_rr (sess, &packet, &pinfo); break; case GST_RTCP_TYPE_SDES: @@ -3190,11 +3491,13 @@ rtp_session_process_rtcp (RTPSession * sess, GstBuffer * buffer, * a proper process function. */ GST_DEBUG ("got RTCP XR packet, but ignored"); break; + case GST_RTCP_TYPE_INVALID: default: - GST_WARNING ("got unknown RTCP packet type: %d", type); + GST_WARNING ("got unknown/invalid RTCP packet type: %d", type); + more = FALSE; break; } - more = gst_rtcp_packet_move_to_next (&packet); + more = more && gst_rtcp_packet_move_to_next (&packet); } gst_rtcp_buffer_unmap (&rtcp); @@ -3213,10 +3516,6 @@ rtp_session_process_rtcp (RTPSession * sess, GstBuffer * buffer, sess->stats.avg_rtcp_packet_size, pinfo.bytes); RTP_SESSION_UNLOCK (sess); - if (has_report) { - g_object_notify_by_pspec (G_OBJECT (sess), properties[PROP_STATS]); - } - pinfo.data = NULL; clean_packet_info (&pinfo); @@ -3745,9 +4044,17 @@ rtp_session_next_timeout (RTPSession * sess, GstClockTime current_time) } else { if (sess->first_rtcp) { GST_DEBUG ("first RTCP packet"); - /* we are called for the first time */ - interval = calculate_rtcp_interval (sess, FALSE, TRUE); - sess->last_rtcp_interval = interval; + /* We have yet to generate our first rtcp packet. Compute the initial + * interval when the next check time is changing. Otherwise, there is + * nothing to do (as the next check time is still in the future and + * computing a new interval will cause the first rtcp to be delayed). + * When interval-based TWCC is enabled, there can be multiple timeouts + * before the initial rtcp time (and thus, without this conditional, + * the initial rtcp will be delayed indefinitely). */ + if (result != sess->next_rtcp_check_time) { + interval = calculate_rtcp_interval (sess, FALSE, TRUE); + sess->last_rtcp_interval = interval; + } } else if (sess->next_rtcp_check_time < current_time) { GST_DEBUG ("old check time expired, getting new timeout"); /* get a new timeout when we need to */ @@ -3782,6 +4089,18 @@ rtp_session_next_timeout (RTPSession * sess, GstClockTime current_time) early_exit: + sess->next_twcc_rtcp_time = rtp_twcc_manager_get_next_timeout (sess->twcc, + current_time); + if (GST_CLOCK_TIME_IS_VALID (sess->next_twcc_rtcp_time)) { + /* Take min(twcc-time, result) */ + if (!GST_CLOCK_TIME_IS_VALID (result) || sess->next_twcc_rtcp_time < result) { + GST_DEBUG ("Using twcc time %" GST_TIME_FORMAT + " instead of %" GST_TIME_FORMAT, + GST_TIME_ARGS (sess->next_twcc_rtcp_time), GST_TIME_ARGS (result)); + result = sess->next_twcc_rtcp_time; + } + } + GST_DEBUG ("current time: %" GST_TIME_FORMAT ", next time: %" GST_TIME_FORMAT, GST_TIME_ARGS (current_time), GST_TIME_ARGS (result)); @@ -3820,7 +4139,7 @@ typedef struct gboolean timeout_inactive_sources; } ReportData; -static void +static gboolean session_start_rtcp (RTPSession * sess, ReportData * data) { GstRTCPPacket *packet = &data->packet; @@ -3832,6 +4151,9 @@ session_start_rtcp (RTPSession * sess, ReportData * data) gst_rtcp_buffer_map (data->rtcp, GST_MAP_READWRITE, rtcp); + if (data->is_early && sess->reduced_size_rtcp) + return FALSE; + if (RTP_SOURCE_IS_SENDER (own) && (!data->is_early || !sess->reduced_size_rtcp || sess->sr_req_pending)) { guint64 ntptime; @@ -3862,6 +4184,8 @@ session_start_rtcp (RTPSession * sess, ReportData * data) gst_rtcp_buffer_add_packet (rtcp, GST_RTCP_TYPE_RR, packet); gst_rtcp_packet_rr_set_ssrc (packet, own->ssrc); } + + return TRUE; } /* construct a Sender or Receiver Report */ @@ -4068,7 +4392,7 @@ session_nack (const gchar * key, RTPSource * source, ReportData * data) } if (i) { - GST_WARNING ("Removing %u expired NACKS", i); + GST_INFO ("Removing %u expired NACKS", i); rtp_source_clear_nacks (source, i); n_nacks -= i; if (n_nacks == 0) @@ -4144,9 +4468,9 @@ session_nack (const gchar * key, RTPSource * source, ReportData * data) data->may_suppress = FALSE; } -/* perform cleanup of sources that timed out */ +/* mark a source that timed out and update its clock-rate */ static void -session_cleanup (const gchar * key, RTPSource * source, ReportData * data) +update_source (const gchar * key, RTPSource * source, ReportData * data) { gboolean remove = FALSE; gboolean byetimeout = FALSE; @@ -4172,6 +4496,11 @@ session_cleanup (const gchar * key, RTPSource * source, ReportData * data) is_sender = RTP_SOURCE_IS_SENDER (source); is_active = RTP_SOURCE_IS_ACTIVE (source); + /* Fetching a new clock rate could release the RTPSession lock, so we need to + * make sure this is done on a copy of the table */ + if (is_sender) + rtp_source_update_clock_rate (source); + /* our own rtcp interval may have been forced low by secondary configuration, * while sender side may still operate with higher interval, * so do not just take our interval to decide on timing out sender, @@ -4200,14 +4529,14 @@ session_cleanup (const gchar * key, RTPSource * source, ReportData * data) * time. */ if (data->current_time > source->bye_time && data->current_time - source->bye_time > sess->stats.bye_timeout) { - GST_DEBUG ("removing BYE source %08x", source->ssrc); + GST_INFO ("removing BYE source %08x", source->ssrc); remove = TRUE; byetimeout = TRUE; } } if (source->internal && source->sent_bye) { - GST_DEBUG ("removing internal source that has sent BYE %08x", source->ssrc); + GST_INFO ("removing internal source that has sent BYE %08x", source->ssrc); remove = TRUE; } @@ -4219,7 +4548,7 @@ session_cleanup (const gchar * key, RTPSource * source, ReportData * data) if (data->current_time > btime) { interval = MAX (binterval * 5, 5 * GST_SECOND); if (data->current_time - btime > interval) { - GST_DEBUG ("removing timeout source %08x, last %" GST_TIME_FORMAT, + GST_INFO ("removing timeout source %08x, last %" GST_TIME_FORMAT, source->ssrc, GST_TIME_ARGS (btime)); if (source->internal) { /* this is an internal source that is not using our suggested ssrc. @@ -4242,7 +4571,10 @@ session_cleanup (const gchar * key, RTPSource * source, ReportData * data) * holds for our own sources. */ if (is_sender) { /* mind old time that might pre-date last time going to PLAYING */ - btime = MAX (source->last_rtp_activity, sess->start_time); + if (source->last_rtp_activity != GST_CLOCK_TIME_NONE) + btime = MAX (source->last_rtp_activity, sess->start_time); + else + btime = sess->start_time; if (data->current_time > btime) { interval = MAX (binterval * 2, 5 * GST_SECOND); if (data->current_time - btime > interval) { @@ -4422,7 +4754,6 @@ is_rtcp_time (RTPSession * sess, GstClockTime current_time, ReportData * data) /* If this is the first RTCP packet, we can reconsider anything based * on the last RTCP send time because there was none. */ - g_warn_if_fail (!data->is_early); data->is_early = FALSE; new_send_time = current_time; } @@ -4456,6 +4787,26 @@ is_rtcp_time (RTPSession * sess, GstClockTime current_time, ReportData * data) } } sess->next_rtcp_check_time = current_time + interval; + } else { + /* + * Even though this is an early request if we haven't sent a regular packet + * for at least 'interval' then we need to include an RR/SR + */ + if (sess->last_rtcp_send_time != GST_CLOCK_TIME_NONE + && interval != GST_CLOCK_TIME_NONE) { + GstClockTime elapsed = current_time - sess->last_rtcp_send_time; + + if (elapsed > interval) { + data->is_early = FALSE; + GST_DEBUG_OBJECT (sess, + "Ajusting early request: current_time %" GST_TIME_FORMAT + " last_rtcp_send_time %" GST_TIME_FORMAT " elapsed %" + GST_TIME_FORMAT " interval %" GST_TIME_FORMAT + " updated is_early = %s", GST_TIME_ARGS (current_time), + GST_TIME_ARGS (sess->last_rtcp_send_time), GST_TIME_ARGS (elapsed), + GST_TIME_ARGS (interval), data->is_early ? "TRUE" : "FALSE"); + } + } } @@ -4475,8 +4826,13 @@ static gboolean remove_closing_sources (const gchar * key, RTPSource * source, ReportData * data) { - if (source->closing) + if (source->closing) { + GST_INFO ("Removing source %08x %p, " + "internal=%u, marked_bye=%u, sent_bye=%u, bye_reason=%s", + source->ssrc, source, source->internal, source->marked_bye, + source->sent_bye, source->bye_reason); return TRUE; + } if (source->send_fir) data->have_fir = TRUE; @@ -4510,7 +4866,8 @@ generate_twcc (const gchar * key, RTPSource * source, ReportData * data) GST_DEBUG ("generating TWCC feedback for source %08x", source->ssrc); - while ((buf = rtp_twcc_manager_get_feedback (sess->twcc, source->ssrc))) { + while ((buf = rtp_twcc_manager_get_feedback (sess->twcc, source->ssrc, + data->current_time))) { ReportOutput *output = g_new (ReportOutput, 1); output->source = g_object_ref (source); output->is_bye = FALSE; @@ -4528,6 +4885,7 @@ generate_rtcp (const gchar * key, RTPSource * source, ReportData * data) gboolean is_bye = FALSE; ReportOutput *output; gboolean sr_req_pending = sess->sr_req_pending; + gboolean is_sr_rr = FALSE; /* only generate RTCP for active internal sources */ if (!source->internal || source->sent_bye) @@ -4546,17 +4904,22 @@ generate_rtcp (const gchar * key, RTPSource * source, ReportData * data) data->source = source; /* open packet */ - session_start_rtcp (sess, data); + is_sr_rr = session_start_rtcp (sess, data); if (source->marked_bye) { /* send BYE */ make_source_bye (sess, source, data); is_bye = TRUE; - } else if (!data->is_early) { - /* loop over all known sources and add report blocks. If we are early, we - * just make a minimal RTCP packet and skip this step */ - g_hash_table_foreach (sess->ssrcs[sess->mask_idx], - (GHFunc) session_report_blocks, data); + } else if (is_sr_rr) { + if (!data->is_early) { + /* loop over all known sources and add report blocks. If we are early, we + * just make a minimal RTCP packet and skip this step */ + g_hash_table_foreach (sess->ssrcs[sess->mask_idx], + (GHFunc) session_report_blocks, data); + } + /* Optionally add profile-specific extension */ + g_signal_emit (sess, rtp_session_signals[SIGNAL_ON_CREATING_SR_RR], 0, + source, &data->packet); } if (!data->has_sdes && (!data->is_early || !sess->reduced_size_rtcp || sr_req_pending)) @@ -4646,6 +5009,32 @@ rtp_session_are_all_sources_bye (RTPSession * sess) return TRUE; } +static gboolean +awoken_for_twcc (RTPSession * sess, GstClockTime current_time) +{ + if (!GST_CLOCK_TIME_IS_VALID (rtp_twcc_manager_get_feedback_interval + (sess->twcc))) { + /* Interval-based TWCC is disabled, so this cannot be a TWCC wake */ + return FALSE; + } + + if (GST_CLOCK_TIME_IS_VALID (sess->next_early_rtcp_time) && + sess->next_early_rtcp_time <= current_time) { + /* Early time is before now, so this may be a non-TWCC wake */ + return FALSE; + } + + if (GST_CLOCK_TIME_IS_VALID (sess->next_rtcp_check_time) && + sess->next_rtcp_check_time <= current_time) { + /* Check time is before now, so this may be a non-TWCC wake */ + return FALSE; + } + + /* Interval-based TWCC is enabled, and neither early nor check times + * are before now, so this must be a TWCC wake */ + return TRUE; +} + /** * rtp_session_on_timeout: * @sess: an #RTPSession @@ -4673,6 +5062,7 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time, GHashTable *table_copy; ReportOutput *output; gboolean all_empty = FALSE; + gboolean twcc_only = FALSE; g_return_val_if_fail (RTP_IS_SESSION (sess), GST_FLOW_ERROR); @@ -4691,6 +5081,7 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time, g_queue_init (&data.output); RTP_SESSION_LOCK (sess); + /* get a new interval, we need this for various cleanups etc */ data.interval = calculate_rtcp_interval (sess, TRUE, sess->first_rtcp); @@ -4711,32 +5102,54 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time, g_object_unref (source); } - sess->conflicting_addresses = - timeout_conflicting_addresses (sess->conflicting_addresses, current_time); + twcc_only = awoken_for_twcc (sess, current_time); + if (!twcc_only) { + sess->conflicting_addresses = + timeout_conflicting_addresses (sess->conflicting_addresses, + current_time); - /* Make a local copy of the hashtable. We need to do this because the - * cleanup stage below releases the session lock. */ - table_copy = g_hash_table_new_full (NULL, NULL, NULL, - (GDestroyNotify) g_object_unref); - g_hash_table_foreach (sess->ssrcs[sess->mask_idx], - (GHFunc) clone_ssrcs_hashtable, table_copy); + /* Make a local copy of the hashtable. We need to do this because the + * update stage below releases the session lock. */ + table_copy = g_hash_table_new_full (NULL, NULL, NULL, + (GDestroyNotify) g_object_unref); + g_hash_table_foreach (sess->ssrcs[sess->mask_idx], + (GHFunc) clone_ssrcs_hashtable, table_copy); - /* Clean up the session, mark the source for removing, this might release the - * session lock. */ - g_hash_table_foreach (table_copy, (GHFunc) session_cleanup, &data); - g_hash_table_destroy (table_copy); + /* Clean up the session, mark the source for removing and update clock-rate, + * this might release the session lock. */ + g_hash_table_foreach (table_copy, (GHFunc) update_source, &data); + g_hash_table_destroy (table_copy); - /* Now remove the marked sources */ - g_hash_table_foreach_remove (sess->ssrcs[sess->mask_idx], - (GHRFunc) remove_closing_sources, &data); + /* Now remove the marked sources */ + g_hash_table_foreach_remove (sess->ssrcs[sess->mask_idx], + (GHRFunc) remove_closing_sources, &data); - /* update point-to-point status */ - session_update_ptp (sess); + /* update point-to-point status */ + session_update_ptp (sess); + } + + /* generate interval-based twcc feedback */ + if (GST_CLOCK_TIME_IS_VALID (rtp_twcc_manager_get_feedback_interval + (sess->twcc))) { + GST_DEBUG ("interval-based twcc. now: %" GST_TIME_FORMAT " twcc-time: %" + GST_TIME_FORMAT, GST_TIME_ARGS (current_time), + GST_TIME_ARGS (sess->next_twcc_rtcp_time)); + if (GST_CLOCK_TIME_IS_VALID (sess->next_twcc_rtcp_time) + && sess->next_twcc_rtcp_time <= current_time) { + GST_DEBUG ("generating twcc"); + g_hash_table_foreach (sess->ssrcs[sess->mask_idx], + (GHFunc) generate_twcc, &data); + sess->next_twcc_rtcp_time = GST_CLOCK_TIME_NONE; + } + } /* see if we need to generate SR or RR packets */ if (!is_rtcp_time (sess, current_time, &data)) goto done; + /* Cannot get here if TWCC only */ + g_assert (!twcc_only); + /* check if all the buffers are empty after generation */ all_empty = TRUE; @@ -4755,7 +5168,12 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time, * session lock. */ g_hash_table_foreach (table_copy, (GHFunc) generate_rtcp, &data); - g_hash_table_foreach (table_copy, (GHFunc) generate_twcc, &data); + /* add twcc feedback if not using interval-based feedback */ + if (!GST_CLOCK_TIME_IS_VALID (rtp_twcc_manager_get_feedback_interval + (sess->twcc))) { + GST_DEBUG ("generating irregular twcc"); + g_hash_table_foreach (table_copy, (GHFunc) generate_twcc, &data); + } /* update the generation for all the sources that have been reported */ g_hash_table_foreach (table_copy, (GHFunc) update_generation, &data); @@ -4786,7 +5204,14 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time, RTP_SESSION_UNLOCK (sess); /* notify about updated statistics */ - g_object_notify_by_pspec (G_OBJECT (sess), properties[PROP_STATS]); + if (!twcc_only) { + if (!GST_CLOCK_TIME_IS_VALID (sess->last_stats_notify_time) || + (GST_CLOCK_DIFF (sess->last_stats_notify_time, current_time) >= + sess->stats_notify_min_interval_ms * GST_MSECOND)) { + g_object_notify_by_pspec (G_OBJECT (sess), properties[PROP_STATS]); + sess->last_stats_notify_time = current_time; + } + } /* push out the RTCP packets */ while ((output = g_queue_pop_head (&data.output))) { @@ -4837,13 +5262,15 @@ rtp_session_on_timeout (RTPSession * sess, GstClockTime current_time, } if (all_empty) - GST_ERROR ("generated empty RTCP messages for all the sources"); + GST_INFO ("generated empty RTCP messages for all the sources"); - /* schedule remaining nacks */ - RTP_SESSION_LOCK (sess); - g_hash_table_foreach (sess->ssrcs[sess->mask_idx], - (GHFunc) schedule_remaining_nacks, &data); - RTP_SESSION_UNLOCK (sess); + if (!twcc_only) { + /* schedule remaining nacks */ + RTP_SESSION_LOCK (sess); + g_hash_table_foreach (sess->ssrcs[sess->mask_idx], + (GHFunc) schedule_remaining_nacks, &data); + RTP_SESSION_UNLOCK (sess); + } return result; } @@ -4862,7 +5289,7 @@ gboolean rtp_session_request_early_rtcp (RTPSession * sess, GstClockTime current_time, GstClockTime max_delay) { - GstClockTime T_dither_max, T_rr, offset = 0; + GstClockTime T_dither_max = 0, T_rr, offset = 0; gboolean ret; gboolean allow_early; @@ -4874,6 +5301,14 @@ rtp_session_request_early_rtcp (RTPSession * sess, GstClockTime current_time, * to be sent */ sess->rtp_profile = GST_RTP_PROFILE_AVPF; + /* special case, a max_delay of 0 means a RTCP-packet should be forced out + instantly, effectively ignoring RFC 4585 section 3.5. Usecases includes + some non-standard ways of using RTCP, like towards Microsoft Lync */ + if (max_delay == 0) { + sess->next_rtcp_check_time = current_time; + goto send_early; + } + /* Check if already requested */ /* RFC 4585 section 3.5.2 step 2 */ if (GST_CLOCK_TIME_IS_VALID (sess->next_early_rtcp_time)) { @@ -4921,9 +5356,7 @@ rtp_session_request_early_rtcp (RTPSession * sess, GstClockTime current_time, /* When there is one auxiliary stream the session can still do point * to point. */ - if (sess->is_doing_ptp) { - T_dither_max = 0; - } else { + if (!sess->is_doing_ptp) { /* Divide by 2 because l = 0.5 */ T_dither_max = T_rr; T_dither_max /= 2; @@ -4986,6 +5419,8 @@ rtp_session_request_early_rtcp (RTPSession * sess, GstClockTime current_time, goto end; } +send_early: + /* RFC 4585 section 3.5.2 step 4b */ if (T_dither_max) { /* Schedule an early transmission later */ @@ -5058,6 +5493,16 @@ rtp_session_send_rtcp (RTPSession * sess, GstClockTime max_delay) return rtp_session_send_rtcp_internal (sess, now, max_delay); } +static void +rtp_session_nack_probe (RTPSession * sess, + guint ssrc, guint pct, GstClockTime duration) +{ + sess->nack_probe_ssrc = ssrc; + sess->nack_probe_pct = pct; + sess->nack_probe_duration = duration; + sess->nack_probe_deadline = GST_CLOCK_TIME_NONE; +} + gboolean rtp_session_request_key_unit (RTPSession * sess, guint32 ssrc, gboolean fir, gint count) @@ -5128,7 +5573,7 @@ rtp_session_request_nack (RTPSession * sess, guint32 ssrc, guint16 seqnum, rtp_source_register_nack (source, seqnum, now + max_delay); RTP_SESSION_UNLOCK (sess); - if (!rtp_session_send_rtcp_internal (sess, now, max_delay)) { + if (!rtp_session_send_rtcp_internal (sess, now, 0)) { GST_DEBUG ("NACK not sent early, sending with next regular RTCP"); } diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h index e48810ed0ef..9970624f0c1 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsession.h @@ -246,6 +246,7 @@ struct _RTPSession { guint probation; guint32 max_dropout_time; guint32 max_misorder_time; + gboolean ssrc_collision_detection; GstRTPProfile rtp_profile; @@ -279,6 +280,7 @@ struct _RTPSession { gboolean allow_early; GstClockTime next_early_rtcp_time; + GstClockTime next_twcc_rtcp_time; gboolean sr_req_pending; gboolean scheduled_bye; @@ -299,6 +301,8 @@ struct _RTPSession { RTPSessionStats stats; RTPSessionStats bye_stats; + GstClockTime last_stats_notify_time; + guint stats_notify_min_interval_ms; gboolean favor_new; GstClockTime rtcp_feedback_retention_window; @@ -310,6 +314,12 @@ struct _RTPSession { gboolean timestamp_sender_reports; + /* nack probe */ + guint32 nack_probe_ssrc; + guint nack_probe_pct; + GstClockTime nack_probe_duration; + GstClockTime nack_probe_deadline; + /* RFC6051 64-bit NTP header extension */ guint8 send_ntp64_ext_id; @@ -319,7 +329,8 @@ struct _RTPSession { /* Transport-wide cc-extension */ RTPTWCCManager *twcc; - RTPTWCCStats *twcc_stats; + GstStructure *rtx_ssrc_map; + GHashTable *rtx_ssrc_to_ssrc; }; /** @@ -333,7 +344,10 @@ struct _RTPSessionClass { GObjectClass parent_class; /* action signals */ + GstStructure* (*get_twcc_windowed_stats) (RTPSession *sess, + GstClockTime stats_window_size, GstClockTime stats_window_delay); RTPSource* (*get_source_by_ssrc) (RTPSession *sess, guint32 ssrc); + void (*nack_probe) (RTPSession *sess, guint ssrc, guint pct, GstClockTime duration); /* signals */ void (*on_new_ssrc) (RTPSession *sess, RTPSource *source); @@ -341,6 +355,8 @@ struct _RTPSessionClass { void (*on_ssrc_validated) (RTPSession *sess, RTPSource *source); void (*on_ssrc_active) (RTPSession *sess, RTPSource *source); void (*on_ssrc_sdes) (RTPSession *sess, RTPSource *source); + void (*on_ssrc_pse) (RTPSession *sess, RTPSource *source, + guint type, GstBuffer *pse, guint rb_count); void (*on_bye_ssrc) (RTPSession *sess, RTPSource *source); void (*on_bye_timeout) (RTPSession *sess, RTPSource *source); void (*on_timeout) (RTPSession *sess, RTPSource *source); @@ -349,6 +365,8 @@ struct _RTPSessionClass { gboolean early); void (*on_app_rtcp) (RTPSession *sess, guint subtype, guint ssrc, const gchar *name, GstBuffer *data); + void (*on_creating_sr_rr) (RTPSession *sess, guint type, RTPSource *source, + GstRTCPPacket * packet); void (*on_feedback_rtcp) (RTPSession *sess, guint type, guint fbtype, guint sender_ssrc, guint media_ssrc, GstBuffer *fci); gboolean (*send_rtcp) (RTPSession *sess, GstClockTime max_delay); diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c index 673f1ad6828..37cbb3fdd4b 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.c @@ -28,6 +28,7 @@ GST_DEBUG_CATEGORY_STATIC (rtp_source_debug); #define GST_CAT_DEFAULT rtp_source_debug +#define NO_CLOCK_RATE_WARNING_THROTTLE 128 #define RTP_MAX_PROBATION_LEN 32 /* signals and args */ @@ -68,6 +69,58 @@ static void rtp_source_set_property (GObject * object, guint prop_id, static void rtp_source_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); +static GQuark quark_application_x_rtp_source_stats; +static GQuark quark_avg_frame_transmission_duration; +static GQuark quark_bitrate; +static GQuark quark_bytes_received; +static GQuark quark_clock_rate; +static GQuark quark_first_rtp_activity; +static GQuark quark_have_rb; +static GQuark quark_have_sr; +static GQuark quark_internal; +static GQuark quark_is_csrc; +static GQuark quark_is_sender; +static GQuark quark_jitter; +static GQuark quark_last_rtp_activity; +static GQuark quark_max_frame_transmission_duration; +static GQuark quark_octets_received; +static GQuark quark_octets_sent; +static GQuark quark_packets_lost; +static GQuark quark_packets_received; +static GQuark quark_packets_sent; +static GQuark quark_rb_dlsr; +static GQuark quark_rb_exthighestseq; +static GQuark quark_rb_fractionlost; +static GQuark quark_rb_jitter; +static GQuark quark_rb_lsr; +static GQuark quark_rb_packetslost; +static GQuark quark_rb_round_trip; +static GQuark quark_rb_ssrc; +static GQuark quark_received_bye; +static GQuark quark_recv_fir_count; +static GQuark quark_recv_nack_count; +static GQuark quark_recv_packet_rate; +static GQuark quark_recv_pli_count; +static GQuark quark_rtcp_from; +static GQuark quark_rtp_from; +static GQuark quark_sent_fir_count; +static GQuark quark_sent_nack_count; +static GQuark quark_sent_pli_count; +static GQuark quark_sent_rb; +static GQuark quark_sent_rb_dlsr; +static GQuark quark_sent_rb_exthighestseq; +static GQuark quark_sent_rb_fractionlost; +static GQuark quark_sent_rb_jitter; +static GQuark quark_sent_rb_lsr; +static GQuark quark_sent_rb_packetslost; +static GQuark quark_seqnum_base; +static GQuark quark_sr_ntptime; +static GQuark quark_sr_octet_count; +static GQuark quark_sr_packet_count; +static GQuark quark_sr_rtptime; +static GQuark quark_ssrc; +static GQuark quark_validated; + /* static guint rtp_source_signals[LAST_SIGNAL] = { 0 }; */ G_DEFINE_TYPE (RTPSource, rtp_source, G_TYPE_OBJECT); @@ -77,6 +130,64 @@ rtp_source_class_init (RTPSourceClass * klass) { GObjectClass *gobject_class; + quark_application_x_rtp_source_stats = + g_quark_from_static_string ("application/x-rtp-source-stats"); + quark_avg_frame_transmission_duration = + g_quark_from_static_string ("avg-frame-transmission-duration"); + quark_bitrate = g_quark_from_static_string ("bitrate"); + quark_bytes_received = g_quark_from_static_string ("bytes-received"); + quark_clock_rate = g_quark_from_static_string ("clock-rate"); + quark_first_rtp_activity = g_quark_from_static_string ("first-rtp-activity"); + quark_have_rb = g_quark_from_static_string ("have-rb"); + quark_have_sr = g_quark_from_static_string ("have-sr"); + quark_internal = g_quark_from_static_string ("internal"); + quark_is_csrc = g_quark_from_static_string ("is-csrc"); + quark_is_sender = g_quark_from_static_string ("is-sender"); + quark_jitter = g_quark_from_static_string ("jitter"); + quark_last_rtp_activity = g_quark_from_static_string ("last-rtp-activity"); + quark_max_frame_transmission_duration = + g_quark_from_static_string ("max-frame-transmission-duration"); + quark_octets_received = g_quark_from_static_string ("octets-received"); + quark_octets_sent = g_quark_from_static_string ("octets-sent"); + quark_packets_lost = g_quark_from_static_string ("packets-lost"); + quark_packets_received = g_quark_from_static_string ("packets-received"); + quark_packets_sent = g_quark_from_static_string ("packets-sent"); + quark_rb_dlsr = g_quark_from_static_string ("rb-dlsr"); + quark_rb_exthighestseq = g_quark_from_static_string ("rb-exthighestseq"); + quark_rb_fractionlost = g_quark_from_static_string ("rb-fractionlost"); + quark_rb_jitter = g_quark_from_static_string ("rb-jitter"); + quark_rb_lsr = g_quark_from_static_string ("rb-lsr"); + quark_rb_packetslost = g_quark_from_static_string ("rb-packetslost"); + quark_rb_round_trip = g_quark_from_static_string ("rb-round-trip"); + quark_rb_ssrc = g_quark_from_static_string ("rb-ssrc"); + quark_received_bye = g_quark_from_static_string ("received-bye"); + quark_recv_fir_count = g_quark_from_static_string ("recv-fir-count"); + quark_recv_nack_count = g_quark_from_static_string ("recv-nack-count"); + quark_recv_packet_rate = g_quark_from_static_string ("recv-packet-rate"); + quark_recv_pli_count = g_quark_from_static_string ("recv-pli-count"); + quark_rtcp_from = g_quark_from_static_string ("rtcp-from"); + quark_rtp_from = g_quark_from_static_string ("rtp-from"); + quark_sent_fir_count = g_quark_from_static_string ("sent-fir-count"); + quark_sent_nack_count = g_quark_from_static_string ("sent-nack-count"); + quark_sent_pli_count = g_quark_from_static_string ("sent-pli-count"); + quark_sent_rb = g_quark_from_static_string ("sent-rb"); + quark_sent_rb_dlsr = g_quark_from_static_string ("sent-rb-dlsr"); + quark_sent_rb_exthighestseq = + g_quark_from_static_string ("sent-rb-exthighestseq"); + quark_sent_rb_fractionlost = + g_quark_from_static_string ("sent-rb-fractionlost"); + quark_sent_rb_jitter = g_quark_from_static_string ("sent-rb-jitter"); + quark_sent_rb_lsr = g_quark_from_static_string ("sent-rb-lsr"); + quark_sent_rb_packetslost = + g_quark_from_static_string ("sent-rb-packetslost"); + quark_seqnum_base = g_quark_from_static_string ("seqnum-base"); + quark_sr_ntptime = g_quark_from_static_string ("sr-ntptime"); + quark_sr_octet_count = g_quark_from_static_string ("sr-octet-count"); + quark_sr_packet_count = g_quark_from_static_string ("sr-packet-count"); + quark_sr_rtptime = g_quark_from_static_string ("sr-rtptime"); + quark_ssrc = g_quark_from_static_string ("ssrc"); + quark_validated = g_quark_from_static_string ("validated"); + gobject_class = (GObjectClass *) klass; gobject_class->finalize = rtp_source_finalize; @@ -152,6 +263,10 @@ rtp_source_class_init (RTPSourceClass * klass) * * "is-sender" G_TYPE_BOOLEAN this source is a sender * * "seqnum-base" G_TYPE_INT first seqnum if known * * "clock-rate" G_TYPE_INT the clock rate of the media + * * "first-rtp-activity" G_TYPE_UINT64 time of first rtp activity + * * "last-rtp-activity" G_TYPE_UINT64 time of latest rtp activity + * * "avg-frame-transmission-duration" G_TYPE_UINT64 weighted average of time it takes to transmit a frame + * * "max-frame-transmission-duration" G_TYPE_UINT64 max time measured it takes to transmit a frame * * The following fields are only present when known. * @@ -271,6 +386,10 @@ rtp_source_reset (RTPSource * src) g_queue_foreach (src->retained_feedback, (GFunc) gst_buffer_unref, NULL); g_queue_clear (src->retained_feedback); src->last_rtptime = -1; + src->first_rtp_activity = GST_CLOCK_TIME_NONE; + src->last_rtp_activity = GST_CLOCK_TIME_NONE; + src->last_ctime = GST_CLOCK_TIME_NONE; + src->frame_ctime = GST_CLOCK_TIME_NONE; src->stats.cycles = -1; src->stats.jitter = 0; @@ -283,6 +402,8 @@ rtp_source_reset (RTPSource * src) src->stats.prev_rtcptime = GST_CLOCK_TIME_NONE; src->stats.last_rtptime = GST_CLOCK_TIME_NONE; src->stats.last_rtcptime = GST_CLOCK_TIME_NONE; + src->stats.avg_frame_transmission_duration = GST_CLOCK_TIME_NONE; + src->stats.max_frame_transmission_duration = 0; g_array_set_size (src->nacks, 0); src->stats.sent_pli_count = 0; @@ -310,6 +431,7 @@ rtp_source_init (RTPSource * src) src->clock_rate = -1; src->packets = g_queue_new (); src->seqnum_offset = -1; + src->no_clock_rate_count = 0; src->retained_feedback = g_queue_new (); src->nacks = g_array_new (FALSE, FALSE, sizeof (guint16)); @@ -388,85 +510,111 @@ rtp_source_create_stats (RTPSource * src) guint32 packet_count = 0; guint32 octet_count = 0; - /* common data for all types of sources */ - s = gst_structure_new ("application/x-rtp-source-stats", - "ssrc", G_TYPE_UINT, (guint) src->ssrc, - "internal", G_TYPE_BOOLEAN, internal, - "validated", G_TYPE_BOOLEAN, src->validated, - "received-bye", G_TYPE_BOOLEAN, src->marked_bye, - "is-csrc", G_TYPE_BOOLEAN, src->is_csrc, - "is-sender", G_TYPE_BOOLEAN, is_sender, - "seqnum-base", G_TYPE_INT, src->seqnum_offset, - "clock-rate", G_TYPE_INT, src->clock_rate, NULL); + s = gst_structure_new_id (quark_application_x_rtp_source_stats, + quark_ssrc, G_TYPE_UINT, (guint) src->ssrc, + quark_internal, G_TYPE_BOOLEAN, internal, + quark_validated, G_TYPE_BOOLEAN, src->validated, + quark_received_bye, G_TYPE_BOOLEAN, src->marked_bye, + quark_is_csrc, G_TYPE_BOOLEAN, src->is_csrc, + quark_is_sender, G_TYPE_BOOLEAN, is_sender, + quark_first_rtp_activity, G_TYPE_UINT64, src->first_rtp_activity, + quark_last_rtp_activity, G_TYPE_UINT64, src->last_rtp_activity, + quark_avg_frame_transmission_duration, G_TYPE_UINT64, + src->stats.avg_frame_transmission_duration, + quark_max_frame_transmission_duration, G_TYPE_UINT64, + src->stats.max_frame_transmission_duration, NULL); /* add address and port */ if (src->rtp_from) { address_str = __g_socket_address_to_string (src->rtp_from); - gst_structure_set (s, "rtp-from", G_TYPE_STRING, address_str, NULL); + gst_structure_id_set (s, quark_rtp_from, G_TYPE_STRING, address_str, NULL); g_free (address_str); } if (src->rtcp_from) { address_str = __g_socket_address_to_string (src->rtcp_from); - gst_structure_set (s, "rtcp-from", G_TYPE_STRING, address_str, NULL); + gst_structure_id_set (s, quark_rtcp_from, G_TYPE_STRING, address_str, NULL); g_free (address_str); } - gst_structure_set (s, - "octets-sent", G_TYPE_UINT64, src->stats.octets_sent, - "packets-sent", G_TYPE_UINT64, src->stats.packets_sent, - "octets-received", G_TYPE_UINT64, src->stats.octets_received, - "packets-received", G_TYPE_UINT64, src->stats.packets_received, - "bytes-received", G_TYPE_UINT64, src->stats.bytes_received, - "bitrate", G_TYPE_UINT64, src->bitrate, - "packets-lost", G_TYPE_INT, - (gint) rtp_stats_get_packets_lost (&src->stats), "jitter", G_TYPE_UINT, - (guint) (src->stats.jitter >> 4), - "sent-pli-count", G_TYPE_UINT, src->stats.sent_pli_count, - "recv-pli-count", G_TYPE_UINT, src->stats.recv_pli_count, - "sent-fir-count", G_TYPE_UINT, src->stats.sent_fir_count, - "recv-fir-count", G_TYPE_UINT, src->stats.recv_fir_count, - "sent-nack-count", G_TYPE_UINT, src->stats.sent_nack_count, - "recv-nack-count", G_TYPE_UINT, src->stats.recv_nack_count, - "recv-packet-rate", G_TYPE_UINT, - gst_rtp_packet_rate_ctx_get (&src->packet_rate_ctx), NULL); + /* is_sender applies to internal sources you send with, but also + the equivalent source on the receiver side */ + if (is_sender) { + gst_structure_id_set (s, + quark_clock_rate, G_TYPE_INT, src->clock_rate, + quark_bitrate, G_TYPE_UINT64, src->bitrate, NULL); + + if (internal) { + gst_structure_id_set (s, + quark_seqnum_base, G_TYPE_INT, src->seqnum_offset, + quark_octets_sent, G_TYPE_UINT64, src->stats.octets_sent, + quark_packets_sent, G_TYPE_UINT64, src->stats.packets_sent, + quark_recv_pli_count, G_TYPE_UINT, src->stats.recv_pli_count, + quark_recv_fir_count, G_TYPE_UINT, src->stats.recv_fir_count, + quark_recv_nack_count, G_TYPE_UINT, src->stats.recv_nack_count, NULL); + } else { + gst_structure_id_set (s, + quark_octets_received, G_TYPE_UINT64, src->stats.octets_received, + quark_packets_received, G_TYPE_UINT64, src->stats.packets_received, + quark_bytes_received, G_TYPE_UINT64, src->stats.bytes_received, + quark_packets_lost, G_TYPE_INT, + (gint) rtp_stats_get_packets_lost (&src->stats), + quark_jitter, G_TYPE_UINT, + (guint) (src->stats.jitter >> 4), + quark_sent_pli_count, G_TYPE_UINT, src->stats.sent_pli_count, + quark_sent_fir_count, G_TYPE_UINT, src->stats.sent_fir_count, + quark_sent_nack_count, G_TYPE_UINT, src->stats.sent_nack_count, + quark_recv_packet_rate, G_TYPE_UINT, + gst_rtp_packet_rate_ctx_get (&src->packet_rate_ctx), NULL); + } + } /* get the last SR. */ have_sr = rtp_source_get_last_sr (src, &time, &ntptime, &rtptime, &packet_count, &octet_count); - gst_structure_set (s, - "have-sr", G_TYPE_BOOLEAN, have_sr, - "sr-ntptime", G_TYPE_UINT64, ntptime, - "sr-rtptime", G_TYPE_UINT, (guint) rtptime, - "sr-octet-count", G_TYPE_UINT, (guint) octet_count, - "sr-packet-count", G_TYPE_UINT, (guint) packet_count, NULL); + gst_structure_id_set (s, quark_have_sr, G_TYPE_BOOLEAN, have_sr, NULL); + if (have_sr) { + gst_structure_id_set (s, + quark_sr_ntptime, G_TYPE_UINT64, ntptime, + quark_sr_rtptime, G_TYPE_UINT, (guint) rtptime, + quark_sr_octet_count, G_TYPE_UINT, (guint) octet_count, + quark_sr_packet_count, G_TYPE_UINT, (guint) packet_count, NULL); + } if (!internal) { /* get the last RB we sent */ - gst_structure_set (s, - "sent-rb", G_TYPE_BOOLEAN, src->last_rr.is_valid, - "sent-rb-fractionlost", G_TYPE_UINT, (guint) src->last_rr.fractionlost, - "sent-rb-packetslost", G_TYPE_INT, (gint) src->last_rr.packetslost, - "sent-rb-exthighestseq", G_TYPE_UINT, - (guint) src->last_rr.exthighestseq, "sent-rb-jitter", G_TYPE_UINT, - (guint) src->last_rr.jitter, "sent-rb-lsr", G_TYPE_UINT, - (guint) src->last_rr.lsr, "sent-rb-dlsr", G_TYPE_UINT, - (guint) src->last_rr.dlsr, NULL); - + gst_structure_id_set (s, + quark_sent_rb, G_TYPE_BOOLEAN, src->last_rr.is_valid, NULL); + if (src->last_rr.is_valid) { + gst_structure_id_set (s, + quark_sent_rb_fractionlost, G_TYPE_UINT, + (guint) src->last_rr.fractionlost, + quark_sent_rb_packetslost, G_TYPE_INT, + (gint) src->last_rr.packetslost, + quark_sent_rb_exthighestseq, G_TYPE_UINT, + (guint) src->last_rr.exthighestseq, + quark_sent_rb_jitter, G_TYPE_UINT, + (guint) src->last_rr.jitter, + quark_sent_rb_lsr, G_TYPE_UINT, + (guint) src->last_rr.lsr, + quark_sent_rb_dlsr, G_TYPE_UINT, (guint) src->last_rr.dlsr, NULL); + } + } else { /* get the last RB */ have_rb = rtp_source_get_last_rb (src, &ssrc, &fractionlost, &packetslost, &exthighestseq, &jitter, &lsr, &dlsr, &round_trip); - - gst_structure_set (s, - "have-rb", G_TYPE_BOOLEAN, have_rb, - "rb-ssrc", G_TYPE_UINT, ssrc, - "rb-fractionlost", G_TYPE_UINT, (guint) fractionlost, - "rb-packetslost", G_TYPE_INT, (gint) packetslost, - "rb-exthighestseq", G_TYPE_UINT, (guint) exthighestseq, - "rb-jitter", G_TYPE_UINT, (guint) jitter, - "rb-lsr", G_TYPE_UINT, (guint) lsr, - "rb-dlsr", G_TYPE_UINT, (guint) dlsr, - "rb-round-trip", G_TYPE_UINT, (guint) round_trip, NULL); + gst_structure_id_set (s, quark_have_rb, G_TYPE_BOOLEAN, have_rb, NULL); + if (have_rb) { + gst_structure_id_set (s, + quark_rb_ssrc, G_TYPE_UINT, ssrc, + quark_rb_fractionlost, G_TYPE_UINT, (guint) fractionlost, + quark_rb_packetslost, G_TYPE_INT, (gint) packetslost, + quark_rb_exthighestseq, G_TYPE_UINT, (guint) exthighestseq, + quark_rb_jitter, G_TYPE_UINT, (guint) jitter, + quark_rb_lsr, G_TYPE_UINT, (guint) lsr, + quark_rb_dlsr, G_TYPE_UINT, (guint) dlsr, + quark_rb_round_trip, G_TYPE_UINT, (guint) round_trip, NULL); + } } return s; @@ -1010,8 +1158,12 @@ calculate_jitter (RTPSource * src, RTPPacketInfo * pinfo) GST_LOG ("SSRC %08x got payload %d", src->ssrc, pinfo->pt); /* check if clock-rate is valid */ - if (src->clock_rate == -1) + if (src->clock_rate == -1) { + src->no_clock_rate_count++; goto no_clock_rate; + } else { + src->no_clock_rate_count = 0; + } rtptime = pinfo->rtptime; @@ -1053,7 +1205,10 @@ calculate_jitter (RTPSource * src, RTPPacketInfo * pinfo) } no_clock_rate: { - GST_WARNING ("cannot get clock-rate for pt %d", pinfo->pt); + if (((src->no_clock_rate_count) % NO_CLOCK_RATE_WARNING_THROTTLE) == 1) { + GST_WARNING ("cannot get clock-rate for pt %d (count=%d)", pinfo->pt, + src->no_clock_rate_count); + } return; } } @@ -1135,6 +1290,40 @@ do_bitrate_estimation (RTPSource * src, GstClockTime running_time, } } + +/* Transmission duration is the time it takes to transmit all packets for a + * given rtptime. If two packets with the same rtptime is sent/received with + * 10 ms apart, the transmission duration is 10ms. If there's only one packet + * with the given rtptime, the transmission duration is 0. */ +static void +calculate_transmission_duration (RTPSource * src, GstClockTime first, + GstClockTime last) +{ + RTPSourceStats *stats; + GstClockTimeDiff duration; + GstClockTime avg; + gint weight; + + stats = &src->stats; + + duration = GST_CLOCK_DIFF (first, last); + stats->max_frame_transmission_duration = MAX (duration, + stats->max_frame_transmission_duration); + + /* Average is calculated by modified moving average with adaptive weight. + * The weight is adaptive because we want to quickly react to an increase in + * duration and at the same time avoid a very fluctuating result. */ + avg = stats->avg_frame_transmission_duration; + if (!GST_CLOCK_TIME_IS_VALID (avg)) + weight = 1; + else if (duration > avg) + weight = 2; + else + weight = 1024; + avg = (duration + (weight - 1) * avg) / weight; + stats->avg_frame_transmission_duration = avg; +} + static gboolean update_receiver_stats (RTPSource * src, RTPPacketInfo * pinfo, gboolean is_receive) @@ -1143,6 +1332,9 @@ update_receiver_stats (RTPSource * src, RTPPacketInfo * pinfo, RTPSourceStats *stats; gint16 delta; gint32 packet_rate, max_dropout, max_misorder; + guint32 rtptime; + guint64 ext_rtptime; + GstClockTime running_time, current_time; stats = &src->stats; @@ -1168,10 +1360,10 @@ update_receiver_stats (RTPSource * src, RTPPacketInfo * pinfo, src->curr_probation = src->probation; } - if (is_receive) { - expected = src->stats.max_seq + 1; - delta = gst_rtp_buffer_compare_seqnum (expected, seqnr); + expected = src->stats.max_seq + 1; + delta = gst_rtp_buffer_compare_seqnum (expected, seqnr); + if (is_receive) { /* if we are still on probation, check seqnum */ if (src->curr_probation) { /* when in probation, we require consecutive seqnums */ @@ -1275,9 +1467,54 @@ update_receiver_stats (RTPSource * src, RTPPacketInfo * pinfo, } } - src->stats.octets_received += pinfo->payload_len; - src->stats.bytes_received += pinfo->bytes; - src->stats.packets_received += pinfo->packets; + /* due to jitter, we might not get the first packet in the stream first. + update the base_seq in that case */ + if (stats->cycles == 0 && seqnr < src->stats.base_seq) { + GST_INFO ("Updating seqnr-base from %u to %u", src->stats.base_seq, seqnr); + src->stats.base_seq = seqnr; + } + + rtptime = pinfo->rtptime; + running_time = pinfo->running_time; + current_time = pinfo->current_time; + + ext_rtptime = src->last_rtptime; + ext_rtptime = gst_rtp_buffer_ext_timestamp (&ext_rtptime, rtptime); + + GST_LOG ("SSRC %08x, RTP %" G_GUINT64_FORMAT ", running_time %" + GST_TIME_FORMAT ", current_time %" GST_TIME_FORMAT, + src->ssrc, ext_rtptime, GST_TIME_ARGS (running_time), + GST_TIME_ARGS (current_time)); + + if (delta >= 0) { + if (ext_rtptime != src->last_rtptime) { + /* New frame */ + if (G_LIKELY (GST_CLOCK_TIME_IS_VALID (src->frame_ctime))) { + calculate_transmission_duration (src, src->frame_ctime, + src->last_ctime); + } + src->frame_ctime = current_time; + } + src->last_ctime = current_time; + } else { + /* Ignore reordered packets for simplicity */ + } + + if (GST_CLOCK_TIME_IS_VALID (running_time)) { + /* we keep track of the last received RTP timestamp and the corresponding + * buffer running_time so that we can use this info when constructing SR reports */ + src->last_rtime = running_time; + src->last_rtptime = ext_rtptime; + } + + if (pinfo->is_list || + !GST_BUFFER_FLAG_IS_SET (pinfo->data, GST_BUFFER_FLAG_GAP)) { + /* skip stats if gap flag is set on buffer */ + stats->octets_received += pinfo->payload_len; + stats->bytes_received += pinfo->bytes; + stats->packets_received += pinfo->packets; + } + /* for the bitrate estimation consider all lower level headers */ src->bytes_received += pinfo->bytes; @@ -1386,10 +1623,6 @@ GstFlowReturn rtp_source_send_rtp (RTPSource * src, RTPPacketInfo * pinfo) { GstFlowReturn result; - GstClockTime running_time; - guint32 rtptime; - guint64 ext_rtptime; - guint64 rt_diff, rtp_diff; g_return_val_if_fail (RTP_IS_SOURCE (src), GST_FLOW_ERROR); @@ -1401,7 +1634,7 @@ rtp_source_send_rtp (RTPSource * src, RTPPacketInfo * pinfo) return GST_FLOW_OK; if (src->pt_set && src->pt != pinfo->pt) { - GST_WARNING ("Changing pt from %u to %u for SSRC %u", src->pt, pinfo->pt, + GST_DEBUG ("Changing pt from %u to %u for SSRC %u", src->pt, pinfo->pt, src->ssrc); } @@ -1413,38 +1646,8 @@ rtp_source_send_rtp (RTPSource * src, RTPPacketInfo * pinfo) src->stats.octets_sent += pinfo->payload_len; src->bytes_sent += pinfo->bytes; - running_time = pinfo->running_time; - - if (GST_CLOCK_TIME_IS_VALID (running_time)) - do_bitrate_estimation (src, running_time, &src->bytes_sent); - - rtptime = pinfo->rtptime; - - ext_rtptime = src->last_rtptime; - ext_rtptime = gst_rtp_buffer_ext_timestamp (&ext_rtptime, rtptime); - - GST_LOG ("SSRC %08x, RTP %" G_GUINT64_FORMAT ", running_time %" - GST_TIME_FORMAT, src->ssrc, ext_rtptime, GST_TIME_ARGS (running_time)); - - if (ext_rtptime > src->last_rtptime) { - rtp_diff = ext_rtptime - src->last_rtptime; - rt_diff = - GST_CLOCK_TIME_IS_VALID (running_time) ? running_time - - src->last_rtime : GST_CLOCK_TIME_NONE; - - /* calc the diff so we can detect drift at the sender. This can also be used - * to guestimate the clock rate if the NTP time is locked to the RTP - * timestamps (as is the case when the capture device is providing the clock). */ - GST_LOG ("SSRC %08x, diff RTP %" G_GUINT64_FORMAT ", diff running_time %" - GST_TIME_FORMAT, src->ssrc, rtp_diff, GST_TIME_ARGS (rt_diff)); - } - - if (GST_CLOCK_TIME_IS_VALID (running_time)) { - /* we keep track of the last received RTP timestamp and the corresponding - * buffer running_time so that we can use this info when constructing SR reports */ - src->last_rtime = running_time; - src->last_rtptime = ext_rtptime; - } + if (GST_CLOCK_TIME_IS_VALID (pinfo->running_time)) + do_bitrate_estimation (src, pinfo->running_time, &src->bytes_sent); /* push packet */ if (!src->callbacks.push_rtp) @@ -1577,6 +1780,24 @@ rtp_source_process_rb (RTPSource * src, guint32 ssrc, guint64 ntpnstime, src->stats.curr_rr = curridx; } +/** + * rtp_source_update_clock_rate: + * @src: a #RTPSource + * + * Update the clock-rate of a source, it fetchs it if not yet set. + */ +void +rtp_source_update_clock_rate (RTPSource * src) +{ + g_return_if_fail (RTP_IS_SOURCE (src)); + + if (src->clock_rate == -1 && src->pt_set) { + GST_INFO ("no clock-rate, getting for pt %u and SSRC %u", src->pt, + src->ssrc); + fetch_caps_for_payload (src, src->pt); + } +} + /** * rtp_source_get_new_sr: * @src: an #RTPSource @@ -1620,12 +1841,6 @@ rtp_source_get_new_sr (RTPSource * src, guint64 ntpnstime, GST_DEBUG ("last_rtime %" GST_TIME_FORMAT ", last_rtptime %" G_GUINT64_FORMAT, GST_TIME_ARGS (src->last_rtime), t_rtp); - if (src->clock_rate == -1 && src->pt_set) { - GST_INFO ("no clock-rate, getting for pt %u and SSRC %u", src->pt, - src->ssrc); - fetch_caps_for_payload (src, src->pt); - } - if (src->clock_rate != -1) { /* get the diff between the clock running_time and the buffer running_time. * This is the elapsed time, as measured against the pipeline clock, between @@ -1646,7 +1861,7 @@ rtp_source_get_new_sr (RTPSource * src, guint64 ntpnstime, t_rtp -= gst_util_uint64_scale_int (diff, src->clock_rate, GST_SECOND); } } else { - GST_WARNING ("no clock-rate, cannot interpolate rtp time for SSRC %u", + GST_WARNING ("no clock-rate, cannot interpolate rtp time for SSRC %08x", src->ssrc); } diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h index f099e4b7925..57ff27c6076 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpsource.h @@ -31,7 +31,7 @@ /* the default number of consecutive RTP packets we need to receive before the * source is considered valid */ -#define RTP_NO_PROBATION 0 +#define RTP_NO_PROBATION 1 #define RTP_DEFAULT_PROBATION 2 #define RTP_SEQ_MOD (1 << 16) @@ -164,11 +164,15 @@ struct _RTPSource { GstCaps *caps; gint clock_rate; gint32 seqnum_offset; + guint no_clock_rate_count; GstClockTime bye_time; GstClockTime last_activity; GstClockTime last_rtp_activity; + GstClockTime first_rtp_activity; + GstClockTime frame_ctime; + GstClockTime last_ctime; GstClockTime last_rtime; GstClockTime last_rtptime; @@ -256,6 +260,8 @@ void rtp_source_process_rb (RTPSource *src, guint32 ssrc, gu gint32 packetslost, guint32 exthighestseq, guint32 jitter, guint32 lsr, guint32 dlsr); +void rtp_source_update_clock_rate (RTPSource *src); + gboolean rtp_source_get_new_sr (RTPSource *src, guint64 ntpnstime, GstClockTime running_time, guint64 *ntptime, guint32 *rtptime, guint32 *packet_count, guint32 *octet_count); diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c index 0f35046f1e8..928846200da 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.c @@ -41,8 +41,9 @@ gst_rtp_packet_rate_ctx_update (RTPPacketRateCtx * ctx, guint16 seqnum, gint diff_seqnum; gint32 new_packet_rate; gint32 base; + gint32 clock_rate = ctx->clock_rate; - if (ctx->clock_rate <= 0) { + if (clock_rate <= 0) { return ctx->avg_packet_rate; } @@ -55,10 +56,10 @@ gst_rtp_packet_rate_ctx_update (RTPPacketRateCtx * ctx, guint16 seqnum, } diff_seqnum = gst_rtp_buffer_compare_seqnum (ctx->last_seqnum, seqnum); - /* Ignore seqnums that are over 15,000 away from the latest one, it's close - * to 2^14 but far enough to avoid any risk of computing error. + /* Ignore seqnums that are not strictly sequential from the last one. + That way we don't taint our estimate with "bad" packets */ - if (diff_seqnum > 15000) + if (diff_seqnum > 1) goto done_but_save; /* Ignore any packet that is in the past, we're only interested in newer @@ -68,7 +69,7 @@ gst_rtp_packet_rate_ctx_update (RTPPacketRateCtx * ctx, guint16 seqnum, goto done; diff_ts = new_ts - ctx->last_ts; - diff_ts = gst_util_uint64_scale_int (diff_ts, GST_SECOND, ctx->clock_rate); + diff_ts = gst_util_uint64_scale_int (diff_ts, GST_SECOND, clock_rate); new_packet_rate = gst_util_uint64_scale (diff_seqnum, GST_SECOND, diff_ts); /* The goal is that higher packet rates "win". @@ -448,233 +449,3 @@ __g_socket_address_to_string (GSocketAddress * addr) return ret; } - -static void -_append_structure_to_value_array (GValueArray * array, GstStructure * s) -{ - GValue *val; - g_value_array_append (array, NULL); - val = g_value_array_get_nth (array, array->n_values - 1); - g_value_init (val, GST_TYPE_STRUCTURE); - g_value_take_boxed (val, s); -} - -static void -_structure_take_value_array (GstStructure * s, - const gchar * field_name, GValueArray * array) -{ - GValue value = G_VALUE_INIT; - g_value_init (&value, G_TYPE_VALUE_ARRAY); - g_value_take_boxed (&value, array); - gst_structure_take_value (s, field_name, &value); - g_value_unset (&value); -} - -GstStructure * -rtp_twcc_stats_get_packets_structure (GArray * twcc_packets) -{ - GstStructure *ret = gst_structure_new_empty ("RTPTWCCPackets"); - GValueArray *array = g_value_array_new (0); - guint i; - - for (i = 0; i < twcc_packets->len; i++) { - RTPTWCCPacket *pkt = &g_array_index (twcc_packets, RTPTWCCPacket, i); - - GstStructure *pkt_s = gst_structure_new ("RTPTWCCPacket", - "seqnum", G_TYPE_UINT, pkt->seqnum, - "local-ts", G_TYPE_UINT64, pkt->local_ts, - "remote-ts", G_TYPE_UINT64, pkt->remote_ts, - "payload-type", G_TYPE_UCHAR, pkt->pt, - "size", G_TYPE_UINT, pkt->size, - "lost", G_TYPE_BOOLEAN, pkt->status == RTP_TWCC_PACKET_STATUS_NOT_RECV, - NULL); - _append_structure_to_value_array (array, pkt_s); - } - - _structure_take_value_array (ret, "packets", array); - return ret; -} - -static void -rtp_twcc_stats_calculate_stats (RTPTWCCStats * stats, GArray * twcc_packets) -{ - guint packets_recv = 0; - guint i; - - for (i = 0; i < twcc_packets->len; i++) { - RTPTWCCPacket *pkt = &g_array_index (twcc_packets, RTPTWCCPacket, i); - - if (pkt->status != RTP_TWCC_PACKET_STATUS_NOT_RECV) - packets_recv++; - - if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts) && - GST_CLOCK_TIME_IS_VALID (stats->last_local_ts)) { - pkt->local_delta = GST_CLOCK_DIFF (stats->last_local_ts, pkt->local_ts); - } - - if (GST_CLOCK_TIME_IS_VALID (pkt->remote_ts) && - GST_CLOCK_TIME_IS_VALID (stats->last_remote_ts)) { - pkt->remote_delta = - GST_CLOCK_DIFF (stats->last_remote_ts, pkt->remote_ts); - } - - if (GST_CLOCK_STIME_IS_VALID (pkt->local_delta) && - GST_CLOCK_STIME_IS_VALID (pkt->remote_delta)) { - pkt->delta_delta = pkt->remote_delta - pkt->local_delta; - } - - stats->last_local_ts = pkt->local_ts; - stats->last_remote_ts = pkt->remote_ts; - } - - stats->packets_sent = twcc_packets->len; - stats->packets_recv = packets_recv; -} - -static gint -_get_window_start_index (RTPTWCCStats * stats, GstClockTime duration, - GstClockTime * local_duration, GstClockTime * remote_duration) -{ - RTPTWCCPacket *last = NULL; - guint i; - - if (stats->packets->len < 2) - return -1; - - for (i = 0; i < stats->packets->len; i++) { - guint start_index = stats->packets->len - 1 - i; - RTPTWCCPacket *pkt = - &g_array_index (stats->packets, RTPTWCCPacket, start_index); - if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts) - && GST_CLOCK_TIME_IS_VALID (pkt->remote_ts)) { - /* first find the last valid packet */ - if (last == NULL) { - last = pkt; - } else { - /* and then get the duration in local ts */ - GstClockTimeDiff ld = GST_CLOCK_DIFF (pkt->local_ts, last->local_ts); - if (ld >= duration) { - *local_duration = ld; - *remote_duration = GST_CLOCK_DIFF (pkt->remote_ts, last->remote_ts); - return start_index; - } - } - } - } - - return -1; -} - -static void -rtp_twcc_stats_calculate_windowed_stats (RTPTWCCStats * stats) -{ - guint i; - gint start_idx; - guint bits_sent = 0; - guint bits_recv = 0; - guint packets_sent = 0; - guint packets_recv = 0; - guint packets_lost; - GstClockTimeDiff delta_delta_sum = 0; - guint delta_delta_count = 0; - GstClockTime local_duration; - GstClockTime remote_duration; - - start_idx = _get_window_start_index (stats, stats->window_size, - &local_duration, &remote_duration); - if (start_idx == -1) { - return; - } - - /* remove the old packets */ - if (start_idx > 0) - g_array_remove_range (stats->packets, 0, start_idx); - - packets_sent = stats->packets->len - 1; - - for (i = 0; i < packets_sent; i++) { - RTPTWCCPacket *pkt = &g_array_index (stats->packets, RTPTWCCPacket, i); - - if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts)) { - bits_sent += pkt->size * 8; - } - - if (GST_CLOCK_TIME_IS_VALID (pkt->remote_ts)) { - bits_recv += pkt->size * 8; - packets_recv++; - } - - if (GST_CLOCK_STIME_IS_VALID (pkt->delta_delta)) { - delta_delta_sum += pkt->delta_delta; - delta_delta_count++; - } - } - - packets_lost = packets_sent - packets_recv; - stats->packet_loss_pct = (packets_lost * 100) / (gfloat) packets_sent; - - if (delta_delta_count) { - GstClockTimeDiff avg_delta_of_delta = delta_delta_sum / delta_delta_count; - if (GST_CLOCK_STIME_IS_VALID (stats->avg_delta_of_delta)) { - stats->avg_delta_of_delta_change = - (avg_delta_of_delta - - stats->avg_delta_of_delta) / (250 * GST_USECOND); - } - stats->avg_delta_of_delta = avg_delta_of_delta; - } - - if (local_duration > 0) - stats->bitrate_sent = - gst_util_uint64_scale (bits_sent, GST_SECOND, local_duration); - if (remote_duration > 0) - stats->bitrate_recv = - gst_util_uint64_scale (bits_recv, GST_SECOND, remote_duration); - - GST_DEBUG ("Got stats: bits_sent: %u, bits_recv: %u, packets_sent = %u, " - "packets_recv: %u, packetlost_pct = %f, sent_bitrate = %u, " - "recv_bitrate = %u, delta-delta-avg = %" GST_STIME_FORMAT ", " - "delta-delta-change: %f", bits_sent, bits_recv, stats->packets_sent, - packets_recv, stats->packet_loss_pct, stats->bitrate_sent, - stats->bitrate_recv, GST_STIME_ARGS (stats->avg_delta_of_delta), - stats->avg_delta_of_delta_change); -} - -RTPTWCCStats * -rtp_twcc_stats_new (void) -{ - RTPTWCCStats *stats = g_new0 (RTPTWCCStats, 1); - stats->packets = g_array_new (FALSE, FALSE, sizeof (RTPTWCCPacket)); - stats->last_local_ts = GST_CLOCK_TIME_NONE; - stats->last_remote_ts = GST_CLOCK_TIME_NONE; - stats->avg_delta_of_delta = GST_CLOCK_STIME_NONE; - stats->window_size = 300 * GST_MSECOND; /* FIXME: could be configurable? */ - return stats; -} - -void -rtp_twcc_stats_free (RTPTWCCStats * stats) -{ - g_array_unref (stats->packets); - g_free (stats); -} - -static GstStructure * -rtp_twcc_stats_get_stats_structure (RTPTWCCStats * stats) -{ - return gst_structure_new ("RTPTWCCStats", - "bitrate-sent", G_TYPE_UINT, stats->bitrate_sent, - "bitrate-recv", G_TYPE_UINT, stats->bitrate_recv, - "packets-sent", G_TYPE_UINT, stats->packets_sent, - "packets-recv", G_TYPE_UINT, stats->packets_recv, - "packet-loss-pct", G_TYPE_DOUBLE, stats->packet_loss_pct, - "avg-delta-of-delta", G_TYPE_INT64, stats->avg_delta_of_delta, NULL); -} - -GstStructure * -rtp_twcc_stats_process_packets (RTPTWCCStats * stats, GArray * twcc_packets) -{ - rtp_twcc_stats_calculate_stats (stats, twcc_packets); - g_array_append_vals (stats->packets, twcc_packets->data, twcc_packets->len); - rtp_twcc_stats_calculate_windowed_stats (stats); - return rtp_twcc_stats_get_stats_structure (stats); -} diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h index 45ad377ee45..d9e955858ec 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtpstats.h @@ -114,6 +114,8 @@ typedef struct { guint16 header_ext_bit_pattern; guint8 ntp64_ext_id; gboolean have_ntp64_ext; + gint32 rtx_osn; + guint32 rtx_ssrc; } RTPPacketInfo; /** @@ -169,6 +171,10 @@ typedef struct { GstClockTime last_rtptime; GstClockTime last_rtcptime; + /* how long it takes to transmit a frame */ + GstClockTime avg_frame_transmission_duration; + GstClockTime max_frame_transmission_duration; + /* sender and receiver reports */ gint curr_rr; RTPReceiverReport rr[2]; @@ -260,27 +266,6 @@ typedef struct { guint nacks_received; } RTPSessionStats; -/** - * RTPTWCCStats: - * - * Stats kept for a session and used to produce TWCC stats. - */ -typedef struct { - GArray *packets; - GstClockTime window_size; - GstClockTime last_local_ts; - GstClockTime last_remote_ts; - - guint bitrate_sent; - guint bitrate_recv; - guint packets_sent; - guint packets_recv; - gfloat packet_loss_pct; - GstClockTimeDiff avg_delta_of_delta; - gfloat avg_delta_of_delta_change; -} RTPTWCCStats; - - void rtp_stats_init_defaults (RTPSessionStats *stats); void rtp_stats_set_bandwidths (RTPSessionStats *stats, @@ -300,10 +285,4 @@ void rtp_stats_set_min_interval (RTPSessionStats *stats, gboolean __g_socket_address_equal (GSocketAddress *a, GSocketAddress *b); gchar * __g_socket_address_to_string (GSocketAddress * addr); -RTPTWCCStats * rtp_twcc_stats_new (void); -void rtp_twcc_stats_free (RTPTWCCStats * stats); -GstStructure * rtp_twcc_stats_process_packets (RTPTWCCStats * stats, - GArray * twcc_packets); -GstStructure * rtp_twcc_stats_get_packets_structure (GArray * twcc_packets); - #endif /* __RTP_STATS_H__ */ diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c index 2b642bff6bb..1719cf8b99c 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.c @@ -17,15 +17,20 @@ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ +#define GLIB_DISABLE_DEPRECATION_WARNINGS + #include "rtptwcc.h" #include #include #include +#include #include "gstrtputils.h" -GST_DEBUG_CATEGORY_EXTERN (rtp_session_debug); -#define GST_CAT_DEFAULT rtp_session_debug +GST_DEBUG_CATEGORY (rtp_twcc_debug); +#define GST_CAT_DEFAULT rtp_twcc_debug + +#define WEIGHT(a, b, w) (((a) * (w)) + ((b) * (1.0 - (w)))) #define TWCC_EXTMAP_STR "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01" @@ -36,6 +41,8 @@ GST_DEBUG_CATEGORY_EXTERN (rtp_session_debug); #define STATUS_VECTOR_MAX_CAPACITY 14 #define STATUS_VECTOR_TWO_BIT_MAX_CAPACITY 7 +#define MAX_PACKETS_PER_FEEDBACK 65536 + typedef enum { RTP_TWCC_CHUNK_TYPE_RUN_LENGTH = 0, @@ -50,6 +57,13 @@ typedef struct guint8 fb_pkt_count[1]; } RTPTWCCHeader; +typedef enum +{ + RTP_TWCC_PACKET_STATUS_NOT_RECV = 0, + RTP_TWCC_PACKET_STATUS_SMALL_DELTA = 1, + RTP_TWCC_PACKET_STATUS_LARGE_NEGATIVE_DELTA = 2, +} RTPTWCCPacketStatus; + typedef struct { GstClockTime ts; @@ -63,18 +77,525 @@ typedef struct typedef struct { - GstClockTime ts; + GstClockTime local_ts; GstClockTime socket_ts; GstClockTime remote_ts; guint16 seqnum; guint8 pt; guint size; gboolean lost; + gint32 rtx_osn; + guint32 rtx_ssrc; } SentPacket; +typedef struct +{ + RTPTWCCPacketStatus status; + guint16 seqnum; + GstClockTime remote_ts; +} ParsedPacket; + +typedef struct +{ + GstClockTime org_ts; + GstClockTime local_ts; + GstClockTime remote_ts; + guint16 seqnum; + guint size; + guint8 pt; + + GstClockTimeDiff local_delta; + GstClockTimeDiff remote_delta; + GstClockTimeDiff delta_delta; + gboolean recovered; +} StatsPacket; + +static void +stats_packet_init_from_sent_packet (StatsPacket * pkt, SentPacket * sent) +{ + pkt->org_ts = sent->local_ts; + pkt->local_ts = sent->local_ts; + pkt->remote_ts = sent->remote_ts; + pkt->seqnum = sent->seqnum; + pkt->size = sent->size; + pkt->pt = sent->pt; + pkt->recovered = FALSE; + pkt->local_delta = GST_CLOCK_STIME_NONE; + pkt->remote_delta = GST_CLOCK_STIME_NONE; + pkt->delta_delta = GST_CLOCK_STIME_NONE; + + /* if we have a socket-timestamp, use that instead */ + if (GST_CLOCK_TIME_IS_VALID (sent->socket_ts)) { + pkt->local_ts = sent->socket_ts; + } +} + +typedef struct +{ + GArray *packets; + gint64 new_packets_idx; + + /* windowed stats */ + guint packets_sent; + guint packets_recv; + guint bitrate_sent; + guint bitrate_recv; + gdouble packet_loss_pct; + gdouble recovery_pct; + gint64 avg_delta_of_delta; + gdouble delta_of_delta_growth; +} TWCCStatsCtx; + +static void +_append_structure_to_value_array (GValueArray * array, GstStructure * s) +{ + GValue *val; + g_value_array_append (array, NULL); + val = g_value_array_get_nth (array, array->n_values - 1); + g_value_init (val, GST_TYPE_STRUCTURE); + g_value_take_boxed (val, s); +} + +static void +_structure_take_value_array (GstStructure * s, + const gchar * field_name, GValueArray * array) +{ + GValue value = G_VALUE_INIT; + g_value_init (&value, G_TYPE_VALUE_ARRAY); + g_value_take_boxed (&value, array); + gst_structure_take_value (s, field_name, &value); + g_value_unset (&value); +} + +static TWCCStatsCtx * +twcc_stats_ctx_new (void) +{ + TWCCStatsCtx *ctx = g_new0 (TWCCStatsCtx, 1); + + ctx->packets = g_array_new (FALSE, FALSE, sizeof (StatsPacket)); + + return ctx; +} + +static void +twcc_stats_ctx_free (TWCCStatsCtx * ctx) +{ + g_array_unref (ctx->packets); + g_free (ctx); +} + +static GstClockTime +twcc_stats_ctx_get_last_local_ts (TWCCStatsCtx * ctx) +{ + GstClockTime ret = GST_CLOCK_TIME_NONE; + StatsPacket *pkt = NULL; + if (ctx->packets->len > 0) + pkt = &g_array_index (ctx->packets, StatsPacket, ctx->packets->len - 1); + if (pkt) + ret = pkt->local_ts; + return ret; +} + +static gboolean +_get_stats_packets_window (GArray * array, + GstClockTimeDiff start_time, GstClockTimeDiff end_time, + guint * start_idx, guint * num_packets) +{ + gboolean ret = FALSE; + guint end_idx = 0; + guint i; + + if (array->len < 2) { + GST_DEBUG ("Not enough starts to do a window"); + return FALSE; + } + + for (i = 0; i < array->len; i++) { + StatsPacket *pkt = &g_array_index (array, StatsPacket, i); + if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts)) { + GstClockTimeDiff offset = GST_CLOCK_DIFF (pkt->local_ts, start_time); + *start_idx = i; + /* positive number here means it is older than our start time */ + if (offset > 0) { + GST_LOG ("Packet #%u is too old: %" + GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt->local_ts)); + } else { + GST_LOG ("Setting first packet in our window to #%u: %" + GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt->local_ts)); + ret = TRUE; + break; + } + } + } + + /* jump out early if we could not find a start_idx */ + if (!ret) { + return FALSE; + } + + ret = FALSE; + for (i = 0; i < array->len - *start_idx - 1; i++) { + guint idx = array->len - 1 - i; + StatsPacket *pkt = &g_array_index (array, StatsPacket, idx); + if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts)) { + GstClockTimeDiff offset = GST_CLOCK_DIFF (pkt->local_ts, end_time); + if (offset >= 0) { + GST_LOG ("Setting last packet in our window to #%u: %" + GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt->local_ts)); + end_idx = idx; + ret = TRUE; + break; + } else { + GST_LOG ("Packet #%u is too new: %" + GST_TIME_FORMAT, pkt->seqnum, GST_TIME_ARGS (pkt->local_ts)); + } + } + } + + /* jump out early if we could not find a window */ + if (!ret) { + return FALSE; + } + + *num_packets = end_idx - *start_idx + 1; + + return ret; +} + +static gboolean +twcc_stats_ctx_calculate_windowed_stats (TWCCStatsCtx * ctx, + GstClockTimeDiff start_time, GstClockTimeDiff end_time) +{ + GArray *packets = ctx->packets; + guint start_idx; + guint packets_sent = 0; + guint packets_recv = 0; + guint packets_recovered = 0; + guint packets_lost = 0; + + /* FIXME: property ? */ + guint max_stats_packets = 1000; + + guint i; + guint bits_sent = 0; + guint bits_recv = 0; + + GstClockTimeDiff delta_delta_sum = 0; + guint delta_delta_count = 0; + GstClockTimeDiff first_delta_delta_sum = 0; + guint first_delta_delta_count = 0; + GstClockTimeDiff last_delta_delta_sum = 0; + guint last_delta_delta_count = 0; + + StatsPacket *first_local_pkt = NULL; + StatsPacket *last_local_pkt = NULL; + StatsPacket *first_remote_pkt = NULL; + StatsPacket *last_remote_pkt = NULL; + + GstClockTimeDiff local_duration = 0; + GstClockTimeDiff remote_duration = 0; + + ctx->packet_loss_pct = 0.0; + ctx->avg_delta_of_delta = 0; + ctx->delta_of_delta_growth = 0.0; + ctx->bitrate_sent = 0; + ctx->bitrate_recv = 0; + ctx->recovery_pct = -1.0; + + gboolean ret = + _get_stats_packets_window (packets, start_time, end_time, &start_idx, + &packets_sent); + if (!ret || packets_sent < 2) { + GST_INFO ("Not enough packets to fill our window yet!"); + return FALSE; + } + + for (i = 0; i < packets_sent; i++) { + StatsPacket *pkt = &g_array_index (packets, StatsPacket, i + start_idx); + GST_LOG ("STATS WINDOW: %u/%u: pkt #%u, pt: %u, size: %u, arrived: %s, " + "local-ts: %" GST_TIME_FORMAT ", remote-ts %" GST_TIME_FORMAT, + i + 1, packets_sent, pkt->seqnum, pkt->pt, pkt->size * 8, + GST_CLOCK_TIME_IS_VALID (pkt->remote_ts) ? "YES" : "NO", + GST_TIME_ARGS (pkt->local_ts), GST_TIME_ARGS (pkt->remote_ts)); + + if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts)) { + /* don't count the bits for the first packet in the window */ + if (first_local_pkt == NULL) { + first_local_pkt = pkt; + } else { + bits_sent += pkt->size * 8; + } + last_local_pkt = pkt; + } + + if (GST_CLOCK_TIME_IS_VALID (pkt->remote_ts)) { + /* don't count the bits for the first packet in the window */ + if (first_remote_pkt == NULL) { + first_remote_pkt = pkt; + } else { + bits_recv += pkt->size * 8; + } + last_remote_pkt = pkt; + packets_recv++; + } else { + GST_LOG ("Packet #%u is lost, recovered=%u", pkt->seqnum, pkt->recovered); + packets_lost++; + if (pkt->recovered) { + packets_recovered++; + } + } + + if (GST_CLOCK_STIME_IS_VALID (pkt->delta_delta)) { + delta_delta_sum += pkt->delta_delta; + delta_delta_count++; + if (i < packets_sent / 2) { + first_delta_delta_sum += pkt->delta_delta; + first_delta_delta_count++; + } else { + last_delta_delta_sum += pkt->delta_delta; + last_delta_delta_count++; + } + } + } + + ctx->packets_sent = packets_sent; + ctx->packets_recv = packets_recv; + + if (first_local_pkt && last_local_pkt) { + local_duration = + GST_CLOCK_DIFF (first_local_pkt->local_ts, last_local_pkt->local_ts); + } + if (first_remote_pkt && last_remote_pkt) { + remote_duration = + GST_CLOCK_DIFF (first_remote_pkt->remote_ts, + last_remote_pkt->remote_ts); + } + + if (packets_sent) + ctx->packet_loss_pct = (packets_lost * 100) / (gfloat) packets_sent; + + if (packets_lost) { + ctx->recovery_pct = (packets_recovered * 100) / (gfloat) packets_lost; + } + + if (delta_delta_count) { + ctx->avg_delta_of_delta = delta_delta_sum / delta_delta_count; + } + + if (first_delta_delta_count && last_delta_delta_count) { + GstClockTimeDiff first_avg = + first_delta_delta_sum / first_delta_delta_count; + GstClockTimeDiff last_avg = last_delta_delta_sum / last_delta_delta_count; + + /* filter out very small numbers */ + first_avg = MAX (first_avg, 100 * GST_USECOND); + last_avg = MAX (last_avg, 100 * GST_USECOND); + ctx->delta_of_delta_growth = (double) last_avg / (double) first_avg; + } + + if (local_duration > 0) { + ctx->bitrate_sent = + gst_util_uint64_scale (bits_sent, GST_SECOND, local_duration); + } + if (remote_duration > 0) { + ctx->bitrate_recv = + gst_util_uint64_scale (bits_recv, GST_SECOND, remote_duration); + } + + GST_INFO ("Got stats: bits_sent: %u, bits_recv: %u, packets_sent = %u, " + "packets_recv: %u, packetlost_pct = %lf, recovery_pct = %lf, " + "local_duration=%" GST_TIME_FORMAT ", remote_duration=%" GST_TIME_FORMAT + ", " "sent_bitrate = %u, " "recv_bitrate = %u, delta-delta-avg = %" + GST_STIME_FORMAT ", delta-delta-growth=%lf", bits_sent, bits_recv, + packets_sent, packets_recv, ctx->packet_loss_pct, ctx->recovery_pct, + GST_TIME_ARGS (local_duration), GST_TIME_ARGS (remote_duration), + ctx->bitrate_sent, ctx->bitrate_recv, + GST_STIME_ARGS (ctx->avg_delta_of_delta), ctx->delta_of_delta_growth); + + /* trim the stats array down to max packets */ + if (packets->len > max_stats_packets) { + g_array_remove_range (packets, 0, packets->len - max_stats_packets); + } + + return TRUE; +} + +static GstStructure * +twcc_stats_ctx_get_structure (TWCCStatsCtx * ctx) +{ + return gst_structure_new ("RTPTWCCStats", + "packets-sent", G_TYPE_UINT, ctx->packets_sent, + "packets-recv", G_TYPE_UINT, ctx->packets_recv, + "bitrate-sent", G_TYPE_UINT, ctx->bitrate_sent, + "bitrate-recv", G_TYPE_UINT, ctx->bitrate_recv, + "packet-loss-pct", G_TYPE_DOUBLE, ctx->packet_loss_pct, + "recovery-pct", G_TYPE_DOUBLE, ctx->recovery_pct, + "avg-delta-of-delta", G_TYPE_INT64, ctx->avg_delta_of_delta, + "delta-of-delta-growth", G_TYPE_DOUBLE, ctx->delta_of_delta_growth, NULL); +} + + +static void +_update_stats_for_packet_idx (GArray * array, gint packet_idx) +{ + StatsPacket *pkt; + StatsPacket *prev; + + /* we need at least 2 packets to do any stats */ + if (array->len < 2) { + GST_DEBUG ("Not yet 2 packets in stats"); + return; + } + + /* we can't do stats on the first packet */ + if (packet_idx == 0) { + GST_DEBUG ("First packet can't have stats calculated"); + return; + } + + /* packet_idx must be inside the array */ + if (array->len <= packet_idx) { + GST_DEBUG ("Packet idx %u is outside of array!", packet_idx); + return; + } + + pkt = &g_array_index (array, StatsPacket, packet_idx); + prev = &g_array_index (array, StatsPacket, packet_idx - 1); + + if (GST_CLOCK_TIME_IS_VALID (pkt->local_ts) && + GST_CLOCK_TIME_IS_VALID (prev->local_ts)) { + pkt->local_delta = GST_CLOCK_DIFF (prev->local_ts, pkt->local_ts); + GST_LOG ("Calculated local_delta for packet #%u to %" GST_STIME_FORMAT, + pkt->seqnum, GST_STIME_ARGS (pkt->local_delta)); + } + + if (GST_CLOCK_TIME_IS_VALID (pkt->remote_ts) && + GST_CLOCK_TIME_IS_VALID (prev->remote_ts)) { + pkt->remote_delta = GST_CLOCK_DIFF (prev->remote_ts, pkt->remote_ts); + GST_LOG ("Calculated remote_delta for packet #%u to %" GST_STIME_FORMAT, + pkt->seqnum, GST_STIME_ARGS (pkt->remote_delta)); + } + + if (GST_CLOCK_STIME_IS_VALID (pkt->local_delta) && + GST_CLOCK_STIME_IS_VALID (pkt->remote_delta)) { + pkt->delta_delta = pkt->remote_delta - pkt->local_delta; + GST_LOG ("Calculated delta-of-delta for packet #%u to %" GST_STIME_FORMAT, + pkt->seqnum, GST_STIME_ARGS (pkt->delta_delta)); + } +} + +static gint +_get_packet_idx_for_seqnum (GArray * array, guint16 seqnum) +{ + guint i; + + for (i = 0; i < array->len; i++) { + StatsPacket *pkt = &g_array_index (array, StatsPacket, i); + if (pkt->seqnum == seqnum) { + return (gint) i; + } + } + + return -1; +} + +/* assumes all seqnum are in order */ +static gint +_get_packet_idx_for_seqnum_fast (GArray * array, guint16 seqnum) +{ + StatsPacket *first; + guint16 idx; + + if (array->len == 0) + return -1; + + first = &g_array_index (array, StatsPacket, 0); + idx = gst_rtp_buffer_compare_seqnum (first->seqnum, seqnum); + + if (idx < array->len) { + StatsPacket *found = &g_array_index (array, StatsPacket, idx); + if (found->seqnum == seqnum) { + return (gint) idx; + } + } + + return -1; +} + +static gint +twcc_stats_ctx_get_packet_idx_for_seqnum (TWCCStatsCtx * ctx, guint16 seqnum) +{ + gint ret; + + /* assumes all packets seqnum are in order */ + ret = _get_packet_idx_for_seqnum_fast (ctx->packets, seqnum); + if (ret != -1) + return ret; + + return _get_packet_idx_for_seqnum (ctx->packets, seqnum); +} + +static StatsPacket * +twcc_stats_ctx_get_packet_for_seqnum (TWCCStatsCtx * ctx, guint16 seqnum) +{ + StatsPacket *ret = NULL; + gint idx = twcc_stats_ctx_get_packet_idx_for_seqnum (ctx, seqnum); + if (idx != -1) + ret = &g_array_index (ctx->packets, StatsPacket, idx); + + return ret; +} + +static void +twcc_stats_ctx_add_packet (TWCCStatsCtx * ctx, StatsPacket * pkt) +{ + StatsPacket *last = NULL; + gint pkt_idx; + + if (ctx->packets->len > 0) + last = &g_array_index (ctx->packets, StatsPacket, ctx->packets->len - 1); + + /* first a quick check to see if we are going forward in seqnum, + in which case we simply append */ + if (last == NULL || pkt->seqnum > last->seqnum) { + GST_LOG ("Appending #%u to stats packets", pkt->seqnum); + g_array_append_val (ctx->packets, *pkt); + /* and update the stats for this packet */ + _update_stats_for_packet_idx (ctx->packets, ctx->packets->len - 1); + return; + } + + /* here we are dealing with a reordered packet, we start by searching for it */ + pkt_idx = twcc_stats_ctx_get_packet_idx_for_seqnum (ctx, pkt->seqnum); + if (pkt_idx != -1) { + StatsPacket *existing = &g_array_index (ctx->packets, StatsPacket, pkt_idx); + /* only update if we don't already have information about this packet, and + this packet brings something new */ + if (!GST_CLOCK_TIME_IS_VALID (existing->remote_ts) && + GST_CLOCK_TIME_IS_VALID (pkt->remote_ts)) { + GST_DEBUG ("Updating stats packet #%u", pkt->seqnum); + g_array_index (ctx->packets, StatsPacket, pkt_idx) = *pkt; + + /* now update stats for this packet and the next one along */ + _update_stats_for_packet_idx (ctx->packets, pkt_idx); + _update_stats_for_packet_idx (ctx->packets, pkt_idx + 1); + } + return; + } +} + +/******************************************************/ + struct _RTPTWCCManager { GObject object; + GMutex recv_lock; + + GHashTable *ssrc_to_seqmap; + GHashTable *pt_to_twcc_ext_id; + + TWCCStatsCtx *stats_ctx; + GHashTable *stats_ctx_by_pt; guint8 send_ext_id; guint8 recv_ext_id; @@ -85,7 +606,6 @@ struct _RTPTWCCManager GArray *recv_packets; guint64 fb_pkt_count; - gint32 last_seqnum; GArray *sent_packets; GArray *parsed_packets; @@ -103,20 +623,46 @@ struct _RTPTWCCManager GstClockTime next_feedback_send_time; GstClockTime feedback_interval; + + GstClockTimeDiff avg_rtt; + GstClockTime last_report_time; + + RTPTWCCManagerCaps caps_cb; + gpointer caps_ud; }; -G_DEFINE_TYPE (RTPTWCCManager, rtp_twcc_manager, G_TYPE_OBJECT); + +static void rtp_twcc_manager_tx_feedback (GstTxFeedback * parent, + guint64 buffer_id, GstClockTime ts); + +static void +_tx_feedback_init (gpointer g_iface, G_GNUC_UNUSED gpointer iface_data) +{ + GstTxFeedbackInterface *iface = g_iface; + iface->tx_feedback = rtp_twcc_manager_tx_feedback; +} + +G_DEFINE_TYPE_WITH_CODE (RTPTWCCManager, rtp_twcc_manager, G_TYPE_OBJECT, + G_IMPLEMENT_INTERFACE (GST_TYPE_TX_FEEDBACK, _tx_feedback_init)); static void rtp_twcc_manager_init (RTPTWCCManager * twcc) { + twcc->ssrc_to_seqmap = g_hash_table_new_full (NULL, NULL, NULL, + (GDestroyNotify) g_hash_table_destroy); + twcc->pt_to_twcc_ext_id = g_hash_table_new (NULL, NULL); + twcc->recv_packets = g_array_new (FALSE, FALSE, sizeof (RecvPacket)); twcc->sent_packets = g_array_new (FALSE, FALSE, sizeof (SentPacket)); - twcc->parsed_packets = g_array_new (FALSE, FALSE, sizeof (RecvPacket)); + twcc->parsed_packets = g_array_new (FALSE, FALSE, sizeof (ParsedPacket)); + g_mutex_init (&twcc->recv_lock); twcc->rtcp_buffers = g_queue_new (); - twcc->last_seqnum = -1; + twcc->stats_ctx = twcc_stats_ctx_new (); + twcc->stats_ctx_by_pt = g_hash_table_new_full (NULL, NULL, + NULL, (GDestroyNotify) twcc_stats_ctx_free); + twcc->recv_media_ssrc = -1; twcc->recv_sender_ssrc = -1; @@ -124,6 +670,7 @@ rtp_twcc_manager_init (RTPTWCCManager * twcc) twcc->feedback_interval = GST_CLOCK_TIME_NONE; twcc->next_feedback_send_time = GST_CLOCK_TIME_NONE; + twcc->last_report_time = GST_CLOCK_TIME_NONE; } static void @@ -131,10 +678,17 @@ rtp_twcc_manager_finalize (GObject * object) { RTPTWCCManager *twcc = RTP_TWCC_MANAGER_CAST (object); + g_hash_table_destroy (twcc->ssrc_to_seqmap); + g_hash_table_destroy (twcc->pt_to_twcc_ext_id); + g_array_unref (twcc->recv_packets); g_array_unref (twcc->sent_packets); g_array_unref (twcc->parsed_packets); g_queue_free_full (twcc->rtcp_buffers, (GDestroyNotify) gst_buffer_unref); + g_mutex_clear (&twcc->recv_lock); + + g_hash_table_destroy (twcc->stats_ctx_by_pt); + twcc_stats_ctx_free (twcc->stats_ctx); G_OBJECT_CLASS (rtp_twcc_manager_parent_class)->finalize (object); } @@ -144,6 +698,8 @@ rtp_twcc_manager_class_init (RTPTWCCManagerClass * klass) { GObjectClass *gobject_class = (GObjectClass *) klass; gobject_class->finalize = rtp_twcc_manager_finalize; + + GST_DEBUG_CATEGORY_INIT (rtp_twcc_debug, "rtptwcc", 0, "RTP TWCC Manager"); } RTPTWCCManager * @@ -156,6 +712,88 @@ rtp_twcc_manager_new (guint mtu) return twcc; } +static TWCCStatsCtx * +_get_ctx_for_pt (RTPTWCCManager * twcc, guint pt) +{ + TWCCStatsCtx *ctx = + g_hash_table_lookup (twcc->stats_ctx_by_pt, GUINT_TO_POINTER (pt)); + if (!ctx) { + ctx = twcc_stats_ctx_new (); + g_hash_table_insert (twcc->stats_ctx_by_pt, GUINT_TO_POINTER (pt), ctx); + } + return ctx; +} + + +static void +_update_stats_with_recovered (RTPTWCCManager * twcc, guint16 seqnum) +{ + TWCCStatsCtx *ctx; + StatsPacket *pkt = + twcc_stats_ctx_get_packet_for_seqnum (twcc->stats_ctx, seqnum); + + if (pkt == NULL) { + GST_INFO ("Could not find seqnum %u", seqnum); + return; + } + + pkt->recovered = TRUE; + + /* now find the equivalent packet in the payload */ + ctx = _get_ctx_for_pt (twcc, pkt->pt); + pkt = twcc_stats_ctx_get_packet_for_seqnum (ctx, seqnum); + + if (pkt) { + pkt->recovered = TRUE; + } +} + +static gint32 +_lookup_seqnum (RTPTWCCManager * twcc, guint32 ssrc, guint16 seqnum) +{ + gint32 ret = -1; + + GHashTable *seq_to_twcc = + g_hash_table_lookup (twcc->ssrc_to_seqmap, GUINT_TO_POINTER (ssrc)); + if (seq_to_twcc) { + ret = + GPOINTER_TO_UINT (g_hash_table_lookup (seq_to_twcc, + GUINT_TO_POINTER (seqnum))); + } + return ret; +} + +static void +_add_packet_to_stats (RTPTWCCManager * twcc, + ParsedPacket * parsed_pkt, SentPacket * sent_pkt) +{ + TWCCStatsCtx *ctx; + StatsPacket pkt; + + stats_packet_init_from_sent_packet (&pkt, sent_pkt); + + /* add packet to the stats context */ + twcc_stats_ctx_add_packet (twcc->stats_ctx, &pkt); + + /* add packet to the payload specific stats context */ + ctx = _get_ctx_for_pt (twcc, pkt.pt); + twcc_stats_ctx_add_packet (ctx, &pkt); + + /* extra check to see if we can confirm the arrival of a recovery packet, + and hence set the "original" packet as recovered */ + if (parsed_pkt->status != RTP_TWCC_PACKET_STATUS_NOT_RECV + && sent_pkt->rtx_osn != -1) { + gint32 recovered_seq = + _lookup_seqnum (twcc, sent_pkt->rtx_ssrc, sent_pkt->rtx_osn); + if (recovered_seq != -1) { + GST_LOG ("RTX Packet %u protects seqnum %d", sent_pkt->seqnum, + recovered_seq); + _update_stats_with_recovered (twcc, recovered_seq); + } + } +} + + static void recv_packet_init (RecvPacket * packet, guint16 seqnum, RTPPacketInfo * pinfo) { @@ -237,12 +875,74 @@ sent_packet_init (SentPacket * packet, guint16 seqnum, RTPPacketInfo * pinfo, GstRTPBuffer * rtp) { packet->seqnum = seqnum; - packet->ts = pinfo->current_time; - packet->size = gst_rtp_buffer_get_payload_len (rtp); + packet->local_ts = pinfo->current_time; + packet->size = pinfo->bytes + 12; /* the reported wireshark size */ packet->pt = gst_rtp_buffer_get_payload_type (rtp); packet->remote_ts = GST_CLOCK_TIME_NONE; packet->socket_ts = GST_CLOCK_TIME_NONE; packet->lost = FALSE; + packet->rtx_osn = pinfo->rtx_osn; + packet->rtx_ssrc = pinfo->rtx_ssrc; +} + +static void +rtp_twcc_manager_register_seqnum (RTPTWCCManager * twcc, + guint32 ssrc, guint16 seqnum, guint16 twcc_seqnum) +{ + GHashTable *seq_to_twcc = + g_hash_table_lookup (twcc->ssrc_to_seqmap, GUINT_TO_POINTER (ssrc)); + if (!seq_to_twcc) { + seq_to_twcc = g_hash_table_new (NULL, NULL); + g_hash_table_insert (twcc->ssrc_to_seqmap, GUINT_TO_POINTER (ssrc), + seq_to_twcc); + } + g_hash_table_insert (seq_to_twcc, GUINT_TO_POINTER (seqnum), + GUINT_TO_POINTER (twcc_seqnum)); + GST_LOG ("Registering OSN: %u to twcc-twcc_seqnum: %u with ssrc: %u", seqnum, + twcc_seqnum, ssrc); +} + +/* Remove old sent packets and keep them under a maximum threshold, as we + can't accumulate them if we don't get a feedback message from the + receiver. */ +static void +_prune_old_sent_packets (RTPTWCCManager * twcc) +{ + guint length; + + if (twcc->sent_packets->len <= MAX_PACKETS_PER_FEEDBACK) + return; + + length = twcc->sent_packets->len - MAX_PACKETS_PER_FEEDBACK; + g_array_remove_range (twcc->sent_packets, 0, length); +} + +/* +* Returns a pointer to the twcc extension header data for the given id, or adds +* it, if it is not present. +* +* Note: we want to override the extension value if it is present. +*/ +static gpointer +_get_twcc_buffer_ext_data (GstRTPBuffer * rtpbuf, guint8 ext_id) +{ + gpointer data = NULL; + guint16 tmp = 0; + gboolean added; + + if (gst_rtp_buffer_get_extension_onebyte_header (rtpbuf, ext_id, 0, &data, + NULL)) { + return data; + } + + added = gst_rtp_buffer_add_extension_onebyte_header (rtpbuf, ext_id, &tmp, + sizeof (tmp)); + if (added) { + gst_rtp_buffer_get_extension_onebyte_header (rtpbuf, ext_id, 0, &data, + NULL); + } + + return data; } static void @@ -252,22 +952,34 @@ _set_twcc_seqnum_data (RTPTWCCManager * twcc, RTPPacketInfo * pinfo, SentPacket packet; GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; gpointer data; + guint16 seqnum; - if (gst_rtp_buffer_map (buf, GST_MAP_READWRITE, &rtp)) { - if (gst_rtp_buffer_get_extension_onebyte_header (&rtp, - ext_id, 0, &data, NULL)) { - guint16 seqnum = twcc->send_seqnum++; - - GST_WRITE_UINT16_BE (data, seqnum); - sent_packet_init (&packet, seqnum, pinfo, &rtp); - g_array_append_val (twcc->sent_packets, packet); + if (!gst_rtp_buffer_map (buf, GST_MAP_READWRITE, &rtp)) { + GST_WARNING ("Couldn't map the buffer %" GST_PTR_FORMAT, buf); + return; + } - GST_LOG ("Send: twcc-seqnum: %u, pt: %u, marker: %d, len: %u, ts: %" - GST_TIME_FORMAT, seqnum, packet.pt, pinfo->marker, packet.size, - GST_TIME_ARGS (pinfo->current_time)); - } + data = _get_twcc_buffer_ext_data (&rtp, ext_id); + if (!data) { gst_rtp_buffer_unmap (&rtp); + return; } + + seqnum = twcc->send_seqnum++; + GST_WRITE_UINT16_BE (data, seqnum); + sent_packet_init (&packet, seqnum, pinfo, &rtp); + gst_rtp_buffer_unmap (&rtp); + g_array_append_val (twcc->sent_packets, packet); + _prune_old_sent_packets (twcc); + rtp_twcc_manager_register_seqnum (twcc, pinfo->ssrc, pinfo->seqnum, seqnum); + + gst_buffer_add_tx_feedback_meta (pinfo->data, seqnum, + GST_TX_FEEDBACK_CAST (twcc)); + + GST_LOG + ("Send: twcc-seqnum: %u, seqnum: %u, pt: %u, marker: %d, size: %u, ts: %" + GST_TIME_FORMAT, packet.seqnum, pinfo->seqnum, packet.pt, pinfo->marker, + packet.size, GST_TIME_ARGS (pinfo->current_time)); } static void @@ -312,6 +1024,20 @@ rtp_twcc_manager_get_recv_twcc_seqnum (RTPTWCCManager * twcc, return val; } +GstClockTime +rtp_twcc_manager_get_next_timeout (RTPTWCCManager * twcc, + GstClockTime current_time) +{ + if (GST_CLOCK_TIME_IS_VALID (twcc->feedback_interval)) { + if (!GST_CLOCK_TIME_IS_VALID (twcc->next_feedback_send_time)) { + /* First time through: initialise feedback time */ + twcc->next_feedback_send_time = current_time + twcc->feedback_interval; + } + return twcc->next_feedback_send_time; + } + return GST_CLOCK_TIME_NONE; +} + static gint _twcc_seqnum_sort (gconstpointer a, gconstpointer b) { @@ -353,7 +1079,6 @@ rtp_twcc_write_run_length_chunk (GArray * packet_chunks, guint len = MIN (run_length - written, 8191); GST_LOG ("Writing a run-length of %u with status %u", len, status); - gst_bit_writer_init_with_data (&writer, (guint8 *) & data, 2, FALSE); gst_bit_writer_put_bits_uint8 (&writer, RTP_TWCC_CHUNK_TYPE_RUN_LENGTH, 1); gst_bit_writer_put_bits_uint8 (&writer, status, 2); @@ -550,8 +1275,12 @@ rtp_twcc_write_chunks (GArray * packet_chunks, /* first write in any preceeding gaps, we use run-length if it would take up more than one chunk (14/7) */ if (pkt->missing_run > packets_per_chunks) { + GST_LOG ("Missing run of %u, using run-length chunk", pkt->missing_run); rtp_twcc_write_run_length_chunk (packet_chunks, RTP_TWCC_PACKET_STATUS_NOT_RECV, pkt->missing_run); + /* for this case we need to set the missing-run to 0, or else + the status vector chunk will use it and write its own version */ + pkt->missing_run = 0; } /* we have a run of the same status, write a run-length chunk and skip @@ -573,8 +1302,10 @@ rtp_twcc_write_chunks (GArray * packet_chunks, chunk_bit_writer_flush (&writer); } +/* must be called with recv_lock */ static void -rtp_twcc_manager_add_fci (RTPTWCCManager * twcc, GstRTCPPacket * packet) +rtp_twcc_manager_add_fci_unlocked (RTPTWCCManager * twcc, + GstRTCPPacket * packet) { RecvPacket *first, *last, *prev; guint16 packet_count; @@ -595,22 +1326,7 @@ rtp_twcc_manager_add_fci (RTPTWCCManager * twcc, GstRTCPPacket * packet) GstClockTimeDiff delta_ts; gint64 delta_ts_rounded; guint8 fb_pkt_count; - - g_array_sort (twcc->recv_packets, _twcc_seqnum_sort); - - /* Quick scan to remove duplicates */ - prev = &g_array_index (twcc->recv_packets, RecvPacket, 0); - for (i = 1; i < twcc->recv_packets->len;) { - RecvPacket *cur = &g_array_index (twcc->recv_packets, RecvPacket, i); - - if (prev->seqnum == cur->seqnum) { - GST_DEBUG ("Removing duplicate packet #%u", cur->seqnum); - g_array_remove_index (twcc->recv_packets, i); - } else { - prev = cur; - i += 1; - } - } + gboolean missing_packets = FALSE; /* get first and last packet */ first = &g_array_index (twcc->recv_packets, RecvPacket, 0); @@ -641,7 +1357,9 @@ rtp_twcc_manager_add_fci (RTPTWCCManager * twcc, GstRTCPPacket * packet) prev = first; for (i = 0; i < twcc->recv_packets->len; i++) { RecvPacket *pkt = &g_array_index (twcc->recv_packets, RecvPacket, i); - if (i != 0) { + if (i == 0) { + pkt->missing_run = 0; + } else { pkt->missing_run = pkt->seqnum - prev->seqnum - 1; } @@ -660,6 +1378,9 @@ rtp_twcc_manager_add_fci (RTPTWCCManager * twcc, GstRTCPPacket * packet) } run_lenght_helper_update (&rlh, pkt); + if (pkt->missing_run > 0) + missing_packets = TRUE; + GST_LOG ("pkt: #%u, ts: %" GST_TIME_FORMAT " ts_rounded: %" GST_TIME_FORMAT " delta_ts: %" GST_STIME_FORMAT @@ -699,11 +1420,22 @@ rtp_twcc_manager_add_fci (RTPTWCCManager * twcc, GstRTCPPacket * packet) GST_MEMDUMP ("full fci:", fci_data, fci_length); g_array_unref (packet_chunks); - g_array_set_size (twcc->recv_packets, 0); + + /* If we have any missing packets in this report, keep the last packet around, + potentially reporting it several times. + This is mimicing Chrome WebRTC behavior. */ + if (missing_packets) { + twcc->recv_packets = + g_array_remove_range (twcc->recv_packets, 0, + twcc->recv_packets->len - 1); + } else { + g_array_set_size (twcc->recv_packets, 0); + } } +/* must be called with the recv_lock */ static void -rtp_twcc_manager_create_feedback (RTPTWCCManager * twcc) +rtp_twcc_manager_create_feedback_unlocked (RTPTWCCManager * twcc) { GstBuffer *buf; GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; @@ -720,7 +1452,7 @@ rtp_twcc_manager_create_feedback (RTPTWCCManager * twcc) gst_rtcp_packet_fb_set_sender_ssrc (&packet, twcc->recv_sender_ssrc); gst_rtcp_packet_fb_set_media_ssrc (&packet, twcc->recv_media_ssrc); - rtp_twcc_manager_add_fci (twcc, &packet); + rtp_twcc_manager_add_fci_unlocked (twcc, &packet); gst_rtcp_buffer_unmap (&rtcp); @@ -754,11 +1486,6 @@ _many_packets_some_lost (RTPTWCCManager * twcc, guint16 seqnum) first = &g_array_index (twcc->recv_packets, RecvPacket, 0); packet_count = seqnum - first->seqnum + 1; - /* If there are a high number of duplicates, we can't use the following - * metrics */ - if (received_packets > packet_count) - return FALSE; - /* check if we lost half of the threshold */ lost_packets = packet_count - received_packets; if (received_packets >= 30 && lost_packets >= 60) @@ -771,6 +1498,17 @@ _many_packets_some_lost (RTPTWCCManager * twcc, guint16 seqnum) return FALSE; } +static gboolean +_find_seqnum_in_recv_packets (RTPTWCCManager * twcc, guint16 seqnum) +{ + for (guint i = 0; i < twcc->recv_packets->len; i++) { + RecvPacket *pkt = &g_array_index (twcc->recv_packets, RecvPacket, i); + if (pkt->seqnum == seqnum) + return TRUE; + } + return FALSE; +} + gboolean rtp_twcc_manager_recv_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo) { @@ -778,17 +1516,20 @@ rtp_twcc_manager_recv_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo) RecvPacket packet; gint32 seqnum; gint diff; + gboolean reordered_packet = FALSE; seqnum = rtp_twcc_manager_get_recv_twcc_seqnum (twcc, pinfo); if (seqnum == -1) return FALSE; + g_mutex_lock (&twcc->recv_lock); + /* if this packet would exceed the capacity of our MTU, we create a feedback with the current packets, and start over with this one */ if (_exceeds_max_packets (twcc, seqnum)) { GST_INFO ("twcc-seqnum: %u would overflow max packets: %u, create feedback" " with current packets", seqnum, twcc->max_packets_per_rtcp); - rtp_twcc_manager_create_feedback (twcc); + rtp_twcc_manager_create_feedback_unlocked (twcc); send_feedback = TRUE; } @@ -796,52 +1537,49 @@ rtp_twcc_manager_recv_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo) if (twcc->recv_media_ssrc == -1) twcc->recv_media_ssrc = pinfo->ssrc; - /* check if we are reordered, and treat it as lost if we already sent - a feedback msg with a higher seqnum. If the diff is huge, treat - it as a restart of a stream */ - diff = gst_rtp_buffer_compare_seqnum (twcc->expected_recv_seqnum, seqnum); - if (twcc->fb_pkt_count > 0 && diff < 0) { - GST_INFO ("Received out of order packet (%u after %u), treating as lost", - seqnum, twcc->expected_recv_seqnum); - return FALSE; + if (twcc->recv_packets->len > 0) { + RecvPacket *last = &g_array_index (twcc->recv_packets, RecvPacket, + twcc->recv_packets->len - 1); + diff = gst_rtp_buffer_compare_seqnum (last->seqnum, seqnum); + if (diff <= 0) { + /* duplicate check */ + if (_find_seqnum_in_recv_packets (twcc, seqnum)) { + GST_INFO ("Received duplicate packet (#%u), dropping", seqnum); + g_mutex_unlock (&twcc->recv_lock); + return FALSE; + } + /* if not duplicate, it is reordered */ + GST_INFO ("Received a reordered packet (#%u)", seqnum); + reordered_packet = TRUE; + } } /* store the packet for Transport-wide RTCP feedback message */ recv_packet_init (&packet, seqnum, pinfo); g_array_append_val (twcc->recv_packets, packet); - twcc->last_seqnum = seqnum; - GST_LOG ("Receive: twcc-seqnum: %u, pt: %u, marker: %d, ts: %" + /* if we received a reordered packet, we need to sort the list */ + if (reordered_packet) + g_array_sort (twcc->recv_packets, _twcc_seqnum_sort); + + GST_LOG ("Receive: twcc-seqnum: #%u, pt: %u, marker: %d, ts: %" GST_TIME_FORMAT, seqnum, pinfo->pt, pinfo->marker, - GST_TIME_ARGS (pinfo->arrival_time)); + GST_TIME_ARGS (packet.ts)); if (!pinfo->marker) twcc->packet_count_no_marker++; - /* are we sending on an interval, or based on marker bit */ - if (GST_CLOCK_TIME_IS_VALID (twcc->feedback_interval)) { - if (!GST_CLOCK_TIME_IS_VALID (twcc->next_feedback_send_time)) - twcc->next_feedback_send_time = - pinfo->running_time + twcc->feedback_interval; - - if (pinfo->running_time >= twcc->next_feedback_send_time) { - GST_LOG ("Generating feedback : Exceeded feedback interval %" - GST_TIME_FORMAT, GST_TIME_ARGS (twcc->feedback_interval)); - rtp_twcc_manager_create_feedback (twcc); - send_feedback = TRUE; - - while (pinfo->running_time >= twcc->next_feedback_send_time) - twcc->next_feedback_send_time += twcc->feedback_interval; - } - } else if (pinfo->marker || _many_packets_some_lost (twcc, seqnum)) { - GST_LOG ("Generating feedback because of %s", - pinfo->marker ? "marker packet" : "many packets some lost"); - rtp_twcc_manager_create_feedback (twcc); + /* Create feedback, if sending based on marker bit */ + if (!GST_CLOCK_TIME_IS_VALID (twcc->feedback_interval) && + (pinfo->marker || _many_packets_some_lost (twcc, seqnum))) { + rtp_twcc_manager_create_feedback_unlocked (twcc); send_feedback = TRUE; twcc->packet_count_no_marker = 0; } + g_mutex_unlock (&twcc->recv_lock); + return send_feedback; } @@ -857,9 +1595,31 @@ _change_rtcp_fb_sender_ssrc (GstBuffer * buf, guint32 sender_ssrc) } GstBuffer * -rtp_twcc_manager_get_feedback (RTPTWCCManager * twcc, guint sender_ssrc) +rtp_twcc_manager_get_feedback (RTPTWCCManager * twcc, guint sender_ssrc, + GstClockTime current_time) { GstBuffer *buf; + + GST_LOG ("considering twcc. now: %" GST_TIME_FORMAT + " twcc-time: %" GST_TIME_FORMAT " packets: %d", + GST_TIME_ARGS (current_time), + GST_TIME_ARGS (twcc->next_feedback_send_time), twcc->recv_packets->len); + + if (GST_CLOCK_TIME_IS_VALID (twcc->feedback_interval) && + GST_CLOCK_TIME_IS_VALID (twcc->next_feedback_send_time) && + twcc->next_feedback_send_time <= current_time) { + /* Sending on a fixed interval: compute next send time */ + while (twcc->next_feedback_send_time <= current_time) + twcc->next_feedback_send_time += twcc->feedback_interval; + + /* Generate feedback, if there is some to send */ + if (twcc->recv_packets->len > 0) { + g_mutex_lock (&twcc->recv_lock); + rtp_twcc_manager_create_feedback_unlocked (twcc); + g_mutex_unlock (&twcc->recv_lock); + } + } + buf = g_queue_pop_head (twcc->rtcp_buffers); if (buf && twcc->recv_sender_ssrc != sender_ssrc) { @@ -870,52 +1630,129 @@ rtp_twcc_manager_get_feedback (RTPTWCCManager * twcc, guint sender_ssrc) return buf; } +static guint8 +get_twcc_ext_id_for_pt (RTPTWCCManager * twcc, guint8 pt) +{ + guint8 twcc_ext_id; + GstCaps *caps; + gpointer value; + + value = g_hash_table_lookup (twcc->pt_to_twcc_ext_id, GUINT_TO_POINTER (pt)); + if (value) + return GPOINTER_TO_UINT (value); + + if (!twcc->caps_cb) + return 0; + + caps = twcc->caps_cb (pt, twcc->caps_ud); + if (!caps) + return 0; + + twcc_ext_id = + gst_rtp_get_extmap_id_for_attribute (gst_caps_get_structure (caps, 0), + TWCC_EXTMAP_STR); + gst_caps_unref (caps); + + g_hash_table_insert (twcc->pt_to_twcc_ext_id, GUINT_TO_POINTER (pt), + GUINT_TO_POINTER (twcc_ext_id)); + GST_LOG ("Added payload (%u) for twcc send ext-id: %u", pt, twcc_ext_id); + + return twcc_ext_id; +} + void rtp_twcc_manager_send_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo) { + guint8 pinfo_twcc_ext_id; + + pinfo_twcc_ext_id = get_twcc_ext_id_for_pt (twcc, pinfo->pt); + + /* save the first valid twcc extid we get from pt */ + if (pinfo_twcc_ext_id > 0 && twcc->send_ext_id == 0) { + twcc->send_ext_id = pinfo_twcc_ext_id; + GST_INFO ("TWCC enabled for send using extension id: %u", + twcc->send_ext_id); + } + if (twcc->send_ext_id == 0) return; + /* the packet info twcc_ext_id should match the parsed one */ + if (pinfo_twcc_ext_id != twcc->send_ext_id) + return; + rtp_twcc_manager_set_send_twcc_seqnum (twcc, pinfo); } static void -_add_twcc_packet (GArray * twcc_packets, guint16 seqnum, guint status) +rtp_twcc_manager_tx_feedback (GstTxFeedback * parent, guint64 buffer_id, + GstClockTime ts) { - RTPTWCCPacket packet; - memset (&packet, 0, sizeof (RTPTWCCPacket)); - packet.local_ts = GST_CLOCK_TIME_NONE; - packet.remote_ts = GST_CLOCK_TIME_NONE; - packet.local_delta = GST_CLOCK_STIME_NONE; - packet.remote_delta = GST_CLOCK_STIME_NONE; - packet.delta_delta = GST_CLOCK_STIME_NONE; + RTPTWCCManager *twcc = RTP_TWCC_MANAGER_CAST (parent); + guint16 seqnum = (guint16) buffer_id; + SentPacket *first = NULL; + SentPacket *pkt = NULL; + gint idx; + + first = &g_array_index (twcc->sent_packets, SentPacket, 0); + if (first == NULL) { + GST_WARNING ("Received a tx-feedback without having sent any packets?!?"); + return; + } + + idx = gst_rtp_buffer_compare_seqnum (first->seqnum, seqnum); + if (idx < 0) { + pkt = + &g_array_index (twcc->sent_packets, SentPacket, + twcc->sent_packets->len - 1); + } else if (idx < twcc->sent_packets->len) { + pkt = &g_array_index (twcc->sent_packets, SentPacket, idx); + } + + if (pkt && pkt->seqnum == seqnum) { + pkt->socket_ts = ts; + GST_LOG ("packet #%u, setting socket-ts %" GST_TIME_FORMAT, + seqnum, GST_TIME_ARGS (ts)); + } else { + GST_WARNING ("Unable to update send-time for twcc-seqnum #%u", seqnum); + } +} + +static void +_add_parsed_packet (GArray * parsed_packets, guint16 seqnum, guint status) +{ + ParsedPacket packet; packet.seqnum = seqnum; packet.status = status; - g_array_append_val (twcc_packets, packet); + packet.remote_ts = GST_CLOCK_TIME_NONE; + g_array_append_val (parsed_packets, packet); + GST_LOG ("Adding parsed packet #%u with status %u", seqnum, status); } static guint -_parse_run_length_chunk (GstBitReader * reader, GArray * twcc_packets, +_parse_run_length_chunk (GstBitReader * reader, GArray * parsed_packets, guint16 seqnum_offset, guint remaining_packets) { - guint16 run_length; - guint8 status_code; + guint16 run_length = 0; + guint8 status_code = 0; guint i; gst_bit_reader_get_bits_uint8 (reader, &status_code, 2); gst_bit_reader_get_bits_uint16 (reader, &run_length, 13); run_length = MIN (remaining_packets, run_length); + GST_LOG ("Found run-length: %u from seqnum: #%u with status code: %u", + run_length, seqnum_offset, status_code); for (i = 0; i < run_length; i++) { - _add_twcc_packet (twcc_packets, seqnum_offset + i, status_code); + _add_parsed_packet (parsed_packets, seqnum_offset + i, status_code); } return run_length; } static guint -_parse_status_vector_chunk (GstBitReader * reader, GArray * twcc_packets, +_parse_status_vector_chunk (GstBitReader * reader, GArray * parsed_packets, guint16 seqnum_offset, guint remaining_packets) { guint8 symbol_size; @@ -926,38 +1763,30 @@ _parse_status_vector_chunk (GstBitReader * reader, GArray * twcc_packets, symbol_size += 1; num_bits = MIN (remaining_packets, 14 / symbol_size); + GST_LOG ("Found status vector for %u from seqnum: #%u with symbol size: %u", + num_bits, seqnum_offset, symbol_size); + for (i = 0; i < num_bits; i++) { guint8 status_code; if (gst_bit_reader_get_bits_uint8 (reader, &status_code, symbol_size)) - _add_twcc_packet (twcc_packets, seqnum_offset + i, status_code); + _add_parsed_packet (parsed_packets, seqnum_offset + i, status_code); } return num_bits; } -/* Remove all locally stored packets that has been reported - back to us */ +/* keep the last 1000 packets... FIXME: do something smarter */ static void -_prune_sent_packets (RTPTWCCManager * twcc, GArray * twcc_packets) +_prune_sent_packets (RTPTWCCManager * twcc) { - SentPacket *first; - RTPTWCCPacket *last; - guint16 last_idx; - - if (twcc_packets->len == 0 || twcc->sent_packets->len == 0) - return; - - first = &g_array_index (twcc->sent_packets, SentPacket, 0); - last = &g_array_index (twcc_packets, RTPTWCCPacket, twcc_packets->len - 1); - - last_idx = last->seqnum - first->seqnum; - - if (last_idx < twcc->sent_packets->len) - g_array_remove_range (twcc->sent_packets, 0, last_idx); + if (twcc->sent_packets->len > 1000) { + g_array_remove_range (twcc->sent_packets, 0, + twcc->sent_packets->len - 1000); + } } static void -_check_for_lost_packets (RTPTWCCManager * twcc, GArray * twcc_packets, +_check_for_lost_packets (RTPTWCCManager * twcc, GArray * parsed_packets, guint16 base_seqnum, guint16 packet_count, guint8 fb_pkt_count) { guint packets_lost; @@ -992,12 +1821,12 @@ _check_for_lost_packets (RTPTWCCManager * twcc, GArray * twcc_packets, if (base_seqnum < twcc->expected_parsed_seqnum) { GST_DEBUG ("twcc seqnum is older than expected (%u < %u)", base_seqnum, twcc->expected_parsed_seqnum); - return; + goto done; } packets_lost = base_seqnum - twcc->expected_parsed_seqnum; for (i = 0; i < packets_lost; i++) { - _add_twcc_packet (twcc_packets, twcc->expected_parsed_seqnum + i, + _add_parsed_packet (parsed_packets, twcc->expected_parsed_seqnum + i, RTP_TWCC_PACKET_STATUS_NOT_RECV); } @@ -1007,11 +1836,24 @@ _check_for_lost_packets (RTPTWCCManager * twcc, GArray * twcc_packets, return; } -GArray * +static void +_add_parsed_packet_to_value_array (GValueArray * array, ParsedPacket * pkt) +{ + _append_structure_to_value_array (array, + gst_structure_new ("RTPTWCCPacket", + "seqnum", G_TYPE_UINT, pkt->seqnum, + "remote-ts", G_TYPE_UINT64, pkt->remote_ts, + "lost", G_TYPE_BOOLEAN, !GST_CLOCK_TIME_IS_VALID (pkt->remote_ts), + NULL)); +} + +GstStructure * rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, - guint8 * fci_data, guint fci_length) + guint8 * fci_data, guint fci_length, GstClockTime current_time) { - GArray *twcc_packets; + GstStructure *ret; + GValueArray *array; + guint16 base_seqnum; guint16 packet_count; GstClockTime base_time; @@ -1021,12 +1863,16 @@ rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, guint fci_parsed; guint i; SentPacket *first_sent_pkt = NULL; + GstClockTimeDiff rtt = GST_CLOCK_STIME_NONE; if (fci_length < 10) { GST_WARNING ("Malformed TWCC RTCP feedback packet"); return NULL; } + ret = gst_structure_new_empty ("RTPTWCCPackets"); + array = g_value_array_new (0); + base_seqnum = GST_READ_UINT16_BE (&fci_data[0]); packet_count = GST_READ_UINT16_BE (&fci_data[2]); base_time = GST_READ_UINT24_BE (&fci_data[4]) * REF_TIME_UNIT; @@ -1036,10 +1882,9 @@ rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, "base_time %" GST_TIME_FORMAT " fb_pkt_count: %u", base_seqnum, packet_count, GST_TIME_ARGS (base_time), fb_pkt_count); - twcc_packets = g_array_sized_new (FALSE, FALSE, - sizeof (RTPTWCCPacket), packet_count); + g_array_set_size (twcc->parsed_packets, 0); - _check_for_lost_packets (twcc, twcc_packets, + _check_for_lost_packets (twcc, twcc->parsed_packets, base_seqnum, packet_count, fb_pkt_count); fci_parsed = 8; @@ -1050,13 +1895,15 @@ rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, guint remaining_packets = packet_count - packets_parsed; gst_bit_reader_get_bits_uint8 (&reader, &chunk_type, 1); + GST_LOG ("Started parse for chunk-type: %u parsed packets: (%u/%u)", + chunk_type, packets_parsed, packet_count); if (chunk_type == RTP_TWCC_CHUNK_TYPE_RUN_LENGTH) { packets_parsed += _parse_run_length_chunk (&reader, - twcc_packets, seqnum_offset, remaining_packets); + twcc->parsed_packets, seqnum_offset, remaining_packets); } else { packets_parsed += _parse_status_vector_chunk (&reader, - twcc_packets, seqnum_offset, remaining_packets); + twcc->parsed_packets, seqnum_offset, remaining_packets); } fci_parsed += 2; } @@ -1065,8 +1912,8 @@ rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, first_sent_pkt = &g_array_index (twcc->sent_packets, SentPacket, 0); ts_rounded = base_time; - for (i = 0; i < twcc_packets->len; i++) { - RTPTWCCPacket *pkt = &g_array_index (twcc_packets, RTPTWCCPacket, i); + for (i = 0; i < twcc->parsed_packets->len; i++) { + ParsedPacket *pkt = &g_array_index (twcc->parsed_packets, ParsedPacket, i); gint16 delta = 0; GstClockTimeDiff delta_ts; @@ -1080,7 +1927,7 @@ rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, if (fci_parsed > fci_length) { GST_WARNING ("Malformed TWCC RTCP feedback packet"); - g_array_set_size (twcc_packets, 0); + g_array_set_size (twcc->parsed_packets, 0); break; } @@ -1094,29 +1941,95 @@ rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, " status: %u", pkt->seqnum, GST_TIME_ARGS (pkt->remote_ts), GST_STIME_ARGS (delta_ts), pkt->status); + _add_parsed_packet_to_value_array (array, pkt); } if (first_sent_pkt) { SentPacket *found = NULL; - guint16 sent_idx = pkt->seqnum - first_sent_pkt->seqnum; + guint16 sent_idx = + gst_rtp_buffer_compare_seqnum (first_sent_pkt->seqnum, pkt->seqnum); if (sent_idx < twcc->sent_packets->len) found = &g_array_index (twcc->sent_packets, SentPacket, sent_idx); if (found && found->seqnum == pkt->seqnum) { - if (GST_CLOCK_TIME_IS_VALID (found->socket_ts)) { - pkt->local_ts = found->socket_ts; - } else { - pkt->local_ts = found->ts; - } - pkt->size = found->size; - pkt->pt = found->pt; + found->remote_ts = pkt->remote_ts; GST_LOG ("matching pkt: #%u with local_ts: %" GST_TIME_FORMAT - " size: %u", pkt->seqnum, GST_TIME_ARGS (pkt->local_ts), pkt->size); + " size: %u, remote-ts: %" GST_TIME_FORMAT, pkt->seqnum, + GST_TIME_ARGS (found->local_ts), + found->size * 8, GST_TIME_ARGS (pkt->remote_ts)); + + _add_packet_to_stats (twcc, pkt, found); + + /* calculate the round-trip time */ + rtt = GST_CLOCK_DIFF (found->local_ts, current_time); + } } } - _prune_sent_packets (twcc, twcc_packets); + if (GST_CLOCK_STIME_IS_VALID (rtt)) + twcc->avg_rtt = WEIGHT (rtt, twcc->avg_rtt, 0.1); + twcc->last_report_time = current_time; + + _prune_sent_packets (twcc); + + _structure_take_value_array (ret, "packets", array); + + return ret; +} + +GstStructure * +rtp_twcc_manager_get_windowed_stats (RTPTWCCManager * twcc, + GstClockTime stats_window_size, GstClockTime stats_window_delay) +{ + GstStructure *ret; + GValueArray *array; + GHashTableIter iter; + gpointer key; + gpointer value; + GstClockTimeDiff start_time; + GstClockTimeDiff end_time; + + GstClockTime last_ts = twcc_stats_ctx_get_last_local_ts (twcc->stats_ctx); + if (!GST_CLOCK_TIME_IS_VALID (last_ts)) + return twcc_stats_ctx_get_structure (twcc->stats_ctx);; + + array = g_value_array_new (0); + end_time = GST_CLOCK_DIFF (stats_window_delay, last_ts); + start_time = end_time - stats_window_size; + + GST_DEBUG_OBJECT (twcc, + "Calculating windowed stats for the window %" GST_STIME_FORMAT + " starting from %" GST_STIME_FORMAT " to: %" GST_STIME_FORMAT, + GST_STIME_ARGS (stats_window_size), GST_STIME_ARGS (start_time), + GST_STIME_ARGS (end_time)); + + twcc_stats_ctx_calculate_windowed_stats (twcc->stats_ctx, start_time, + end_time); + ret = twcc_stats_ctx_get_structure (twcc->stats_ctx); + GST_LOG ("Full stats: %" GST_PTR_FORMAT, ret); + + g_hash_table_iter_init (&iter, twcc->stats_ctx_by_pt); + while (g_hash_table_iter_next (&iter, &key, &value)) { + GstStructure *s; + guint pt = GPOINTER_TO_UINT (key); + TWCCStatsCtx *ctx = value; + twcc_stats_ctx_calculate_windowed_stats (ctx, start_time, end_time); + s = twcc_stats_ctx_get_structure (ctx); + gst_structure_set (s, "pt", G_TYPE_UINT, pt, NULL); + _append_structure_to_value_array (array, s); + GST_LOG ("Stats for pt %u: %" GST_PTR_FORMAT, pt, s); + } + + _structure_take_value_array (ret, "payload-stats", array); - return twcc_packets; + return ret; } + +void +rtp_twcc_manager_set_callback (RTPTWCCManager * twcc, RTPTWCCManagerCaps cb, + gpointer user_data) +{ + twcc->caps_cb = cb; + twcc->caps_ud = user_data; +} \ No newline at end of file diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.h b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.h index a826e9a4c9a..d15e6976265 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.h +++ b/subprojects/gst-plugins-good/gst/rtpmanager/rtptwcc.h @@ -25,32 +25,20 @@ #include #include "rtpstats.h" -typedef struct _RTPTWCCPacket RTPTWCCPacket; -typedef enum _RTPTWCCPacketStatus RTPTWCCPacketStatus; - G_DECLARE_FINAL_TYPE (RTPTWCCManager, rtp_twcc_manager, RTP, TWCC_MANAGER, GObject) #define RTP_TYPE_TWCC_MANAGER (rtp_twcc_manager_get_type()) #define RTP_TWCC_MANAGER_CAST(obj) ((RTPTWCCManager *)(obj)) -enum _RTPTWCCPacketStatus -{ - RTP_TWCC_PACKET_STATUS_NOT_RECV = 0, - RTP_TWCC_PACKET_STATUS_SMALL_DELTA = 1, - RTP_TWCC_PACKET_STATUS_LARGE_NEGATIVE_DELTA = 2, -}; - -struct _RTPTWCCPacket -{ - GstClockTime local_ts; - GstClockTime remote_ts; - GstClockTimeDiff local_delta; - GstClockTimeDiff remote_delta; - GstClockTimeDiff delta_delta; - RTPTWCCPacketStatus status; - guint16 seqnum; - guint size; - guint8 pt; -}; +/** + * RTPTWCCManagerCaps: + * @payload: the payload + * @user_data: user data specified when registering + * + * This callback will be called when @twcc needs the caps of @payload. + * + * Returns: the caps of @payload. + */ +typedef GstCaps * (*RTPTWCCManagerCaps) (guint8 payload, gpointer user_data); RTPTWCCManager * rtp_twcc_manager_new (guint mtu); @@ -63,6 +51,8 @@ void rtp_twcc_manager_set_mtu (RTPTWCCManager * twcc, guint mtu); void rtp_twcc_manager_set_feedback_interval (RTPTWCCManager * twcc, GstClockTime feedback_interval); GstClockTime rtp_twcc_manager_get_feedback_interval (RTPTWCCManager * twcc); +GstClockTime rtp_twcc_manager_get_next_timeout (RTPTWCCManager * twcc, + GstClockTime current_time); gboolean rtp_twcc_manager_recv_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo); @@ -70,9 +60,15 @@ void rtp_twcc_manager_send_packet (RTPTWCCManager * twcc, RTPPacketInfo * pinfo); GstBuffer * rtp_twcc_manager_get_feedback (RTPTWCCManager * twcc, - guint32 sender_ssrc); + guint32 sender_ssrc, GstClockTime current_time); + +GstStructure * rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, + guint8 * fci_data, guint fci_length, GstClockTime current_time); + +GstStructure * rtp_twcc_manager_get_windowed_stats (RTPTWCCManager * twcc, + GstClockTime stats_window_size, GstClockTime stats_window_delay); -GArray * rtp_twcc_manager_parse_fci (RTPTWCCManager * twcc, - guint8 * fci_data, guint fci_length); +void rtp_twcc_manager_set_callback (RTPTWCCManager * twcc, + RTPTWCCManagerCaps cb, gpointer user_data); #endif /* __RTP_TWCC_H__ */ diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/tokenbucket.c b/subprojects/gst-plugins-good/gst/rtpmanager/tokenbucket.c new file mode 100644 index 00000000000..6a4366e64c1 --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtpmanager/tokenbucket.c @@ -0,0 +1,145 @@ +/* GStreamer +* Copyright (C) 2021 Pexip (http://pexip.com/) +* @author: Havard Graff +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Library General Public +* License as published by the Free Software Foundation; either +* version 2 of the License, or (at your option) any later version. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Library General Public License for more details. +* +* You should have received a copy of the GNU Library General Public +* License along with this library; if not, write to the +* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, +* Boston, MA 02110-1301, USA. +*/ +#include "tokenbucket.h" + +void +token_bucket_reset (TokenBucket * tb) +{ + tb->prev_time = GST_CLOCK_TIME_NONE; + tb->bucket_size = 0; +} + +void +token_bucket_set_bps (TokenBucket * tb, gint64 bps) +{ + tb->bps = bps; +} + +void +token_bucket_set_max_bucket_size (TokenBucket * tb, gint64 max_bucket_size) +{ + tb->max_bucket_size = max_bucket_size; +} + +void +token_bucket_init (TokenBucket * tb, gint64 bps, gint max_bucket_size) +{ + token_bucket_reset (tb); + token_bucket_set_bps (tb, bps); + token_bucket_set_max_bucket_size (tb, max_bucket_size); +} + +void +token_bucket_add_tokens (TokenBucket * tb, GstClockTime now) +{ + gint64 tokens = 0; + GstClockTimeDiff elapsed_time = 0; + + if (!GST_CLOCK_TIME_IS_VALID (now)) + return; + + /* get the elapsed time */ + if (GST_CLOCK_TIME_IS_VALID (tb->prev_time)) { + if (now < tb->prev_time) { + GST_INFO ("We have already produced tokens for this time " + "(%" GST_TIME_FORMAT " < %" GST_TIME_FORMAT ")", + GST_TIME_ARGS (now), GST_TIME_ARGS (tb->prev_time)); + } else { + elapsed_time = GST_CLOCK_DIFF (tb->prev_time, now); + } + } + + tb->prev_time = now; + + /* check for umlimited bps and fill the bucket if that is the case */ + if (tb->bps == -1) { + if (tb->max_bucket_size != -1) + tb->bucket_size = tb->max_bucket_size; + return; + } + + /* no bitrate, no bits to add */ + if (tb->bps == 0) { + return; + } + + /* no extra time to add tokens for */ + if (elapsed_time == 0) { + return; + } + + /* calculate number of tokens and add them to the bucket */ + tokens = gst_util_uint64_scale_round (elapsed_time, tb->bps, GST_SECOND); + tb->bucket_size += tokens; + if (tb->max_bucket_size != -1 && tb->bucket_size > tb->max_bucket_size) { + tb->bucket_size = tb->max_bucket_size; + } + GST_LOG ("Added %" G_GINT64_FORMAT " tokens to bucket " + "(contains %" G_GINT64_FORMAT " tokens)", tokens, tb->bucket_size); + + GST_LOG ("Elapsed time: %" GST_TIME_FORMAT " produces %" G_GINT64_FORMAT + " tokens, new prev_time: %" GST_TIME_FORMAT, + GST_TIME_ARGS (elapsed_time), tokens, GST_TIME_ARGS (tb->prev_time)); +} + +GstClockTime +token_bucket_get_missing_tokens_time (TokenBucket * tb, gint tokens) +{ + GstClockTimeDiff ret; + gint missing_tokens; + + /* unlimited bits per second and unlimited bucket size + means tokens are always available */ + if (tb->bps == -1 && tb->max_bucket_size == -1) + return 0; + + /* we we have room in our bucket, no need to wait */ + if (tb->bucket_size >= tokens) + return 0; + + /* lets not divide by 0 */ + if (tb->bps == 0) + return 0; + + missing_tokens = tokens - tb->bucket_size; + ret = gst_util_uint64_scale (GST_SECOND, missing_tokens, tb->bps); + + return ret; +} + +gboolean +token_bucket_take_tokens (TokenBucket * tb, gint tokens, gboolean force) +{ + /* unlimited bits per second and unlimited bucket size + means tokens are always available */ + if (tb->bps == -1 && tb->max_bucket_size == -1) + return TRUE; + + /* if we force it, or we have enough tokens, remove + the tokens from the bucket */ + if (force || tb->bucket_size >= tokens) { + GST_LOG ("Removing %u tokens from bucket (%" G_GINT64_FORMAT ")", + tokens, tb->bucket_size); + tb->bucket_size -= tokens; + return TRUE; + } + + return FALSE; +} diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/tokenbucket.h b/subprojects/gst-plugins-good/gst/rtpmanager/tokenbucket.h new file mode 100644 index 00000000000..2178c245690 --- /dev/null +++ b/subprojects/gst-plugins-good/gst/rtpmanager/tokenbucket.h @@ -0,0 +1,43 @@ +/* GStreamer +* Copyright (C) 2021 Pexip (http://pexip.com/) +* @author: Havard Graff +* +* This library is free software; you can redistribute it and/or +* modify it under the terms of the GNU Library General Public +* License as published by the Free Software Foundation; either +* version 2 of the License, or (at your option) any later version. +* +* This library is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +* Library General Public License for more details. +* +* You should have received a copy of the GNU Library General Public +* License along with this library; if not, write to the +* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, +* Boston, MA 02110-1301, USA. +*/ +#ifndef __TOKEN_BUCKET_H__ +#define __TOKEN_BUCKET_H__ + +#include + +typedef struct _TokenBucket TokenBucket; + +struct _TokenBucket +{ + gint64 max_bucket_size; + gint64 bps; + GstClockTime prev_time; + gint64 bucket_size; +}; + +void token_bucket_init (TokenBucket * tb, gint64 bps, gint max_bucket_size); +void token_bucket_set_bps (TokenBucket * tb, gint64 bps); +void token_bucket_set_max_bucket_size (TokenBucket * tb, gint64 bps); +void token_bucket_reset (TokenBucket * tb); +void token_bucket_add_tokens (TokenBucket * tb, GstClockTime now); +GstClockTime token_bucket_get_missing_tokens_time (TokenBucket * tb, gint tokens); +gboolean token_bucket_take_tokens (TokenBucket * tb, gint tokens, gboolean force); + +#endif /* __TOKEN_BUCKET_H__ */ diff --git a/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c b/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c index 864594d025a..a54f3e50537 100644 --- a/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c +++ b/subprojects/gst-plugins-good/gst/rtsp/gstrtspsrc.c @@ -5162,7 +5162,7 @@ gst_rtspsrc_configure_caps (GstRTSPSrc * src, GstSegment * segment, gst_caps_set_simple (caps, "clock-base", G_TYPE_UINT, (guint) stream->timebase, NULL); if (stream->seqbase != -1) - gst_caps_set_simple (caps, "seqnum-base", G_TYPE_UINT, + gst_caps_set_simple (caps, "seqnum-offset", G_TYPE_UINT, (guint) stream->seqbase, NULL); gst_caps_set_simple (caps, "npt-start", G_TYPE_UINT64, start, NULL); if (stop != -1) @@ -8847,7 +8847,7 @@ clear_rtp_base (GstRTSPSrc * src, GstRTSPStream * stream) item->caps = gst_caps_make_writable (item->caps); s = gst_caps_get_structure (item->caps, 0); - gst_structure_remove_fields (s, "clock-base", "seqnum-base", NULL); + gst_structure_remove_fields (s, "clock-base", "seqnum-offset", NULL); if (item->pt == stream->default_pt && stream->udpsrc[0]) g_object_set (stream->udpsrc[0], "caps", item->caps, NULL); } diff --git a/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c b/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c index 527beeac870..f5504c37774 100644 --- a/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c +++ b/subprojects/gst-plugins-good/gst/udp/gstmultiudpsink.c @@ -681,14 +681,14 @@ gst_multiudpsink_send_messages (GstMultiUDPSink * sink, GSocket * socket, skip = 1; if (msg_size > UDP_MAX_SIZE) { if (!sent_max_size_warning) { - GST_ELEMENT_WARNING (sink, RESOURCE, WRITE, + GST_ELEMENT_INFO (sink, RESOURCE, WRITE, ("Attempting to send a UDP packets larger than maximum size " "(%u > %d)", msg_size, UDP_MAX_SIZE), ("Reason: %s", err ? err->message : "unknown reason")); sent_max_size_warning = FALSE; } } else { - GST_ELEMENT_WARNING (sink, RESOURCE, WRITE, + GST_ELEMENT_INFO (sink, RESOURCE, WRITE, ("Error sending UDP packets"), ("client %s, reason: %s", gst_udp_address_get_string (msg->address, astr, sizeof (astr)), (err != NULL) ? err->message : "unknown reason")); @@ -714,6 +714,27 @@ gst_multiudpsink_send_messages (GstMultiUDPSink * sink, GSocket * socket, return GST_FLOW_OK; } +static void +_set_time_on_buffers (GstMultiUDPSink * sink, GstBuffer ** buffers, + guint num_buffers) +{ + GstClock *clock = GST_ELEMENT_CLOCK (sink); + GstClockTime now; + guint i; + + if (clock == NULL) + return; + + now = gst_clock_get_time (clock); + + for (i = 0; i < num_buffers; i++) { + GstTxFeedbackMeta *meta = gst_buffer_get_tx_feedback_meta (buffers[i]); + if (meta) { + gst_tx_feedback_meta_set_tx_time (meta, now); + } + } +} + static GstFlowReturn gst_multiudpsink_render_buffers (GstMultiUDPSink * sink, GstBuffer ** buffers, guint num_buffers, guint8 * mem_nums, guint total_mem_num) @@ -832,6 +853,9 @@ gst_multiudpsink_render_buffers (GstMultiUDPSink * sink, GstBuffer ** buffers, if (flow_ret != GST_FLOW_OK) goto cancelled; + + _set_time_on_buffers (sink, buffers, num_buffers); + /* now update stats */ g_mutex_lock (&sink->client_lock); @@ -843,6 +867,7 @@ gst_multiudpsink_render_buffers (GstMultiUDPSink * sink, GstBuffer ** buffers, bytes_sent = msgs[i * num_buffers + j].bytes_sent; + client->bytes_sent += bytes_sent; client->packets_sent++; sink->bytes_served += bytes_sent; diff --git a/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c b/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c index 4870fb1a07c..496b9126e80 100644 --- a/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c +++ b/subprojects/gst-plugins-good/sys/osxaudio/gstosxaudiodeviceprovider.c @@ -54,6 +54,14 @@ G_DEFINE_TYPE (GstOsxAudioDeviceProvider, gst_osx_audio_device_provider, static GList *gst_osx_audio_device_provider_probe (GstDeviceProvider * provider); +static gboolean gst_osx_audio_device_provider_start (GstDeviceProvider * provider); +static void gst_osx_audio_device_provider_stop (GstDeviceProvider * provider); +static OSStatus gst_osx_audio_device_change_cb(AudioObjectID inObjectID, + UInt32 inNumberAddresses, + const AudioObjectPropertyAddress* inAddresses, + void* inClientData); +static void +gst_osx_audio_device_provider_update_devices (GstOsxAudioDeviceProvider * provider); static void gst_osx_audio_device_provider_class_init (GstOsxAudioDeviceProviderClass * @@ -62,6 +70,8 @@ gst_osx_audio_device_provider_class_init (GstOsxAudioDeviceProviderClass * GstDeviceProviderClass *dm_class = GST_DEVICE_PROVIDER_CLASS (klass); dm_class->probe = gst_osx_audio_device_provider_probe; + dm_class->start = gst_osx_audio_device_provider_start; + dm_class->stop = gst_osx_audio_device_provider_stop; gst_device_provider_class_set_static_metadata (dm_class, "OSX Audio Device Provider", "Source/Sink/Audio", @@ -69,11 +79,103 @@ gst_osx_audio_device_provider_class_init (GstOsxAudioDeviceProviderClass * "Hyunjun Ko "); } +static OSStatus gst_osx_audio_device_change_cb(AudioObjectID inObjectID, + guint32 inNumberAddresses, + const AudioObjectPropertyAddress* inAddresses, + void* userdata) { + GstOsxAudioDeviceProvider * provider = (GstOsxAudioDeviceProvider *) userdata; + + for (guint32 i = 0; i < inNumberAddresses; i++) { + switch (inAddresses[i].mSelector) { + case kAudioHardwarePropertyDefaultInputDevice: + gst_osx_audio_device_provider_update_devices(provider); + break; + case kAudioHardwarePropertyDefaultOutputDevice: + gst_osx_audio_device_provider_update_devices(provider); + break; + case kAudioHardwarePropertyDevices: + gst_osx_audio_device_provider_update_devices(provider); + break; + default: + break; + } + } + return noErr; +} + static void gst_osx_audio_device_provider_init (GstOsxAudioDeviceProvider * provider) { } +static gboolean +gst_osx_audio_device_provider_start (GstDeviceProvider * provider) +{ + GstOsxAudioDeviceProvider *self = GST_OSX_AUDIO_DEVICE_PROVIDER (provider); + + // Register callbacks for the following AudioObjectIDs + AudioObjectID event_ids[] = {kAudioHardwarePropertyDevices, kAudioHardwarePropertyDefaultInputDevice, kAudioHardwarePropertyDefaultOutputDevice}; + + for (size_t i = 0; i < sizeof(event_ids) / sizeof(event_ids[0]); i++){ + AudioObjectPropertyAddress deviceListAddr = { + .mSelector = event_ids[i], + .mScope = kAudioObjectPropertyScopeGlobal, + .mElement = kAudioObjectPropertyElementMain + }; + + OSStatus err = + AudioObjectAddPropertyListener(kAudioObjectSystemObject, + &deviceListAddr, + gst_osx_audio_device_change_cb, + (void *)self); + + if (err != noErr) { + GST_ERROR("Failed to register AudioObjectAddPropertyListener(%u) %d", event_ids[i], err); + return FALSE; + } + } + + /* baseclass will not call probe() once it's started, but we can get + * notification only add/remove or change case. To this manually */ + GList *devices = gst_osx_audio_device_provider_probe(provider); + if (devices) { + GList *iter; + for (iter = devices; iter; iter = g_list_next (iter)) { + gst_device_provider_device_add (provider, GST_DEVICE (iter->data)); + } + g_list_free (devices); + } + + return TRUE; +} + +static void +gst_osx_audio_device_provider_stop (GstDeviceProvider * provider) +{ + GstOsxAudioDeviceProvider *self = GST_OSX_AUDIO_DEVICE_PROVIDER (provider); + + // De-register callbacks for the following AudioObjectIDs + AudioObjectID event_ids[] = {kAudioHardwarePropertyDevices, kAudioHardwarePropertyDefaultInputDevice, kAudioHardwarePropertyDefaultOutputDevice}; + + for (size_t i = 0; i < sizeof(event_ids) / sizeof(event_ids[0]); i++){ + AudioObjectPropertyAddress deviceListAddr = { + .mSelector = event_ids[i], + .mScope = kAudioObjectPropertyScopeGlobal, + .mElement = kAudioObjectPropertyElementMain + }; + + OSStatus err = + AudioObjectRemovePropertyListener(kAudioObjectSystemObject, + &deviceListAddr, + gst_osx_audio_device_change_cb, + (void *)self); + + if (err != noErr) { + GST_ERROR("Failed to de-register AudioObjectAddPropertyListener(%u) %d", event_ids[i], err); + } + } +} + static GstOsxAudioDevice * gst_osx_audio_device_provider_probe_device (GstOsxAudioDeviceProvider * provider, AudioDeviceID device_id, const gchar * device_name, @@ -231,6 +333,41 @@ _audio_system_get_devices (gint * ndevices) return devices; } +static gboolean _audio_system_device_is_default(AudioObjectPropertySelector selector, AudioDeviceID queriedDeviceID) +{ + OSStatus error = noErr; + AudioDeviceID deviceID = 0; + AudioObjectPropertyAddress propertyAddress; + UInt32 propertySize; + + //sets which property to check + propertyAddress.mSelector = selector; + propertyAddress.mScope = kAudioObjectPropertyScopeGlobal; + propertyAddress.mElement = 0; + propertySize = sizeof(AudioDeviceID); + + //gets property (system output device) + error = AudioObjectGetPropertyData( kAudioObjectSystemObject, + &propertyAddress, + 0, + NULL, + &propertySize, + &deviceID); + if (error) + return FALSE; + return deviceID == queriedDeviceID; +} + +static gboolean _audio_system_device_is_default_input(AudioDeviceID queriedDeviceID) +{ + return _audio_system_device_is_default(kAudioHardwarePropertyDefaultInputDevice, queriedDeviceID); +} + +static gboolean _audio_system_device_is_default_output(AudioDeviceID queriedDeviceID) +{ + return _audio_system_device_is_default(kAudioHardwarePropertyDefaultOutputDevice, queriedDeviceID); +} + static void gst_osx_audio_device_provider_probe_internal (GstOsxAudioDeviceProvider * self, gboolean is_src, AudioDeviceID * osx_devices, gint ndevices, @@ -309,6 +446,99 @@ gst_osx_audio_device_provider_probe (GstDeviceProvider * provider) return devices; } +static gboolean +gst_osx_audio_device_is_in_list (GList * list, GstDevice * device) +{ + GList *iter; + GstStructure *s; + AudioDeviceID device_id; + gboolean device_is_default; + gboolean found = FALSE; + + s = gst_device_get_properties (device); + g_assert (s); + g_assert(gst_structure_get_int (s, "device-id", &device_id) == TRUE); + g_assert(gst_structure_get_boolean (s, "is-default", &device_is_default) == TRUE); + + for (iter = list; iter; iter = g_list_next (iter)) { + GstStructure *other_s; + AudioDeviceID other_device_id; + gboolean other_device_is_default; + + other_s = gst_device_get_properties (GST_DEVICE (iter->data)); + g_assert (other_s); + + g_assert(gst_structure_get_int (other_s, "device-id", &other_device_id) == TRUE); + g_assert(gst_structure_get_boolean (other_s, "is-default", &other_device_is_default) == TRUE); + + if (device_id == other_device_id && device_is_default == other_device_is_default) { + found = TRUE; + } + + gst_structure_free (other_s); + if (found) + break; + } + + gst_structure_free (s); + + return found; +} + +static void +gst_osx_audio_device_provider_update_devices (GstOsxAudioDeviceProvider * self) +{ + GstDeviceProvider *provider = GST_DEVICE_PROVIDER_CAST (self); + GList *prev_devices = NULL; + GList *new_devices = NULL; + GList *to_add = NULL; + GList *to_remove = NULL; + GList *iter; + + GST_OBJECT_LOCK (self); + prev_devices = g_list_copy_deep (provider->devices, + (GCopyFunc) gst_object_ref, NULL); + GST_OBJECT_UNLOCK (self); + + new_devices = gst_osx_audio_device_provider_probe (provider); + + /* Ownership of GstDevice for gst_device_provider_device_add() + * and gst_device_provider_device_remove() is a bit complicated. + * Remove floating reference here for things to be clear */ + for (iter = new_devices; iter; iter = g_list_next (iter)) + gst_object_ref_sink (iter->data); + + /* Check newly added devices */ + for (iter = new_devices; iter; iter = g_list_next (iter)) { + if (!gst_osx_audio_device_is_in_list (prev_devices, GST_DEVICE (iter->data))) { + to_add = g_list_prepend (to_add, gst_object_ref (iter->data)); + } + } + + /* Check removed device */ + for (iter = prev_devices; iter; iter = g_list_next (iter)) { + if (!gst_osx_audio_device_is_in_list (new_devices, GST_DEVICE (iter->data))) { + to_remove = g_list_prepend (to_remove, gst_object_ref (iter->data)); + } + } + + for (iter = to_remove; iter; iter = g_list_next (iter)) + gst_device_provider_device_remove (provider, GST_DEVICE (iter->data)); + + for (iter = to_add; iter; iter = g_list_next (iter)) + gst_device_provider_device_add (provider, GST_DEVICE (iter->data)); + + if (prev_devices) + g_list_free_full (prev_devices, (GDestroyNotify) gst_object_unref); + + if (to_add) + g_list_free_full (to_add, (GDestroyNotify) gst_object_unref); + + if (to_remove) + g_list_free_full (to_remove, (GDestroyNotify) gst_object_unref); +} + + enum { PROP_DEVICE_ID = 1, @@ -368,6 +598,7 @@ gst_osx_audio_device_new (AudioDeviceID device_id, const gchar * device_name, g_return_val_if_fail (device_id > 0, NULL); g_return_val_if_fail (device_name, NULL); + gboolean is_default = FALSE; switch (type) { case GST_OSX_AUDIO_DEVICE_TYPE_SOURCE: element_name = "osxaudiosrc"; @@ -376,7 +607,7 @@ gst_osx_audio_device_new (AudioDeviceID device_id, const gchar * device_name, template_caps = gst_static_pad_template_get_caps (&src_factory); caps = gst_core_audio_probe_caps (core_audio, template_caps); gst_caps_unref (template_caps); - + is_default = _audio_system_device_is_default_input(device_id); break; case GST_OSX_AUDIO_DEVICE_TYPE_SINK: element_name = "osxaudiosink"; @@ -385,6 +616,7 @@ gst_osx_audio_device_new (AudioDeviceID device_id, const gchar * device_name, template_caps = gst_static_pad_template_get_caps (&sink_factory); caps = gst_core_audio_probe_caps (core_audio, template_caps); gst_caps_unref (template_caps); + is_default = _audio_system_device_is_default_output(device_id); break; default: @@ -392,12 +624,15 @@ gst_osx_audio_device_new (AudioDeviceID device_id, const gchar * device_name, break; } + GstStructure *props = gst_structure_new ("osxaudiodevice-proplist", + "device-id", G_TYPE_INT, device_id, + "is-default", G_TYPE_BOOLEAN, is_default, NULL); gstdev = g_object_new (GST_TYPE_OSX_AUDIO_DEVICE, "device-id", device_id, "display-name", device_name, "caps", caps, "device-class", - klass, NULL); + klass, "properties", props, NULL); gstdev->element = element_name; - + gst_structure_free (props); return gstdev; } diff --git a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2deviceprovider.c b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2deviceprovider.c index 5eeec7dc8b1..a3bbbccf934 100644 --- a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2deviceprovider.c +++ b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2deviceprovider.c @@ -218,7 +218,7 @@ gst_v4l2_device_provider_probe (GstDeviceProvider * provider) if (device) { gst_object_ref_sink (device); - devices = g_list_prepend (devices, device); + devices = g_list_append (devices, device); } } diff --git a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2src.c b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2src.c index 07f8718794b..f711e1ac94c 100644 --- a/subprojects/gst-plugins-good/sys/v4l2/gstv4l2src.c +++ b/subprojects/gst-plugins-good/sys/v4l2/gstv4l2src.c @@ -1128,6 +1128,21 @@ gst_v4l2src_change_state (GstElement * element, GstStateChange transition) gst_v4l2_error (v4l2src, &error); return GST_STATE_CHANGE_FAILURE; } + + /* the device might still busy even after open() succeds, so try also to + set the format, it will return an EBUSY error if the device is + already acquired by another process */ + GstCaps *caps = gst_v4l2_object_get_caps (obj, NULL); + caps = gst_caps_make_writable (caps); + gboolean set_format = gst_v4l2src_set_format (v4l2src, caps, &error); + gst_caps_unref (caps); + + if (!set_format) { + gst_v4l2_error (v4l2src, &error); + gst_v4l2_object_close (obj); + return GST_STATE_CHANGE_FAILURE; + } + break; default: break; diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpbin.c b/subprojects/gst-plugins-good/tests/check/elements/rtpbin.c index 179bcb423f0..0c63e6318c6 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpbin.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpbin.c @@ -27,6 +27,16 @@ #include #include +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_VALGRIND +# include +#else +# define RUNNING_ON_VALGRIND 0 +#endif + GST_START_TEST (test_pads) { GstElement *element; @@ -133,8 +143,7 @@ init_data (CleanupData * data) static void clean_data (CleanupData * data) { - g_list_foreach (data->pads, (GFunc) gst_object_unref, NULL); - g_list_free (data->pads); + g_list_free_full (data->pads, gst_object_unref); g_mutex_clear (&data->lock); g_cond_clear (&data->cond); if (data->caps) @@ -185,7 +194,8 @@ chain_rtp_packet (GstPad * pad, CleanupData * data) } static GstFlowReturn -dummy_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) +dummy_chain (G_GNUC_UNUSED GstPad * pad, G_GNUC_UNUSED GstObject * parent, + GstBuffer * buffer) { gst_buffer_unref (buffer); @@ -214,7 +224,8 @@ make_sinkpad (CleanupData * data) } static void -pad_added_cb (GstElement * rtpbin, GstPad * pad, CleanupData * data) +pad_added_cb (G_GNUC_UNUSED GstElement * rtpbin, GstPad * pad, + CleanupData * data) { GstPad *sinkpad; @@ -236,7 +247,8 @@ pad_added_cb (GstElement * rtpbin, GstPad * pad, CleanupData * data) } static void -pad_removed_cb (GstElement * rtpbin, GstPad * pad, CleanupData * data) +pad_removed_cb (G_GNUC_UNUSED GstElement * rtpbin, GstPad * pad, + CleanupData * data) { GST_DEBUG ("pad removed %s:%s\n", GST_DEBUG_PAD_NAME (pad)); @@ -259,10 +271,14 @@ GST_START_TEST (test_cleanup_recv) GstStateChangeReturn ret; GstFlowReturn res; gint count = 2; + GstClock *clock; init_data (&data); + clock = gst_system_clock_obtain (); rtpbin = gst_element_factory_make ("rtpbin", "rtpbin"); + gst_element_set_clock (rtpbin, clock); + gst_object_unref (clock); g_signal_connect (rtpbin, "pad-added", (GCallback) pad_added_cb, &data); g_signal_connect (rtpbin, "pad-removed", (GCallback) pad_removed_cb, &data); @@ -331,10 +347,14 @@ GST_START_TEST (test_cleanup_recv2) GstStateChangeReturn ret; GstFlowReturn res; gint count = 2; + GstClock *clock; init_data (&data); + clock = gst_system_clock_obtain (); rtpbin = gst_element_factory_make ("rtpbin", "rtpbin"); + gst_element_set_clock (rtpbin, clock); + gst_object_unref (clock); g_signal_connect (rtpbin, "pad-added", (GCallback) pad_added_cb, &data); g_signal_connect (rtpbin, "pad-removed", (GCallback) pad_removed_cb, &data); @@ -441,7 +461,7 @@ GST_START_TEST (test_request_pad_by_template_name) GST_END_TEST; static GstElement * -encoder_cb (GstElement * rtpbin, guint sessid, GstElement * bin) +encoder_cb (G_GNUC_UNUSED GstElement * rtpbin, guint sessid, GstElement * bin) { GstPad *srcpad, *sinkpad; @@ -458,7 +478,7 @@ encoder_cb (GstElement * rtpbin, guint sessid, GstElement * bin) } static GstElement * -encoder_cb2 (GstElement * rtpbin, guint sessid, GstElement * bin) +encoder_cb2 (G_GNUC_UNUSED GstElement * rtpbin, guint sessid, GstElement * bin) { GstPad *srcpad, *sinkpad; @@ -518,7 +538,8 @@ GST_START_TEST (test_encoder) GST_END_TEST; static GstElement * -decoder_cb (GstElement * rtpbin, guint sessid, gpointer user_data) +decoder_cb (G_GNUC_UNUSED GstElement * rtpbin, G_GNUC_UNUSED guint sessid, + G_GNUC_UNUSED gpointer user_data) { GstElement *bin; GstPad *srcpad, *sinkpad; @@ -574,7 +595,8 @@ GST_START_TEST (test_decoder) GST_END_TEST; static GstElement * -aux_sender_cb (GstElement * rtpbin, guint sessid, gpointer user_data) +aux_sender_cb (G_GNUC_UNUSED GstElement * rtpbin, G_GNUC_UNUSED guint sessid, + gpointer user_data) { GstElement *bin; GstPad *srcpad, *sinkpad; @@ -652,7 +674,8 @@ GST_START_TEST (test_aux_sender) GST_END_TEST; static GstElement * -aux_receiver_cb (GstElement * rtpbin, guint sessid, gpointer user_data) +aux_receiver_cb (G_GNUC_UNUSED GstElement * rtpbin, G_GNUC_UNUSED guint sessid, + G_GNUC_UNUSED gpointer user_data) { GstElement *bin; GstPad *srcpad, *sinkpad; @@ -941,8 +964,11 @@ _pad_added (G_GNUC_UNUSED GstElement * rtpbin, GstPad * pad, GstHarness * h) GST_START_TEST (test_quick_shutdown) { guint r; + guint repeats = 1000; + if (RUNNING_ON_VALGRIND) + repeats = 2; - for (r = 0; r < 1000; r++) { + for (r = 0; r < repeats; r++) { guint i; GstHarness *h = gst_harness_new_with_padnames ("rtpbin", "recv_rtp_sink_0", NULL); @@ -969,6 +995,81 @@ GST_START_TEST (test_quick_shutdown) GST_END_TEST; +static GstBuffer * +generate_rtcp_sr_buffer (guint ssrc) +{ + GstBuffer *buf; + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + GstRTCPPacket packet; + + buf = gst_rtcp_buffer_new (1000); + fail_unless (gst_rtcp_buffer_map (buf, GST_MAP_READWRITE, &rtcp)); + fail_unless (gst_rtcp_buffer_add_packet (&rtcp, GST_RTCP_TYPE_SR, &packet)); + gst_rtcp_packet_sr_set_sender_info (&packet, ssrc, 0, 0, 1, 1); + gst_rtcp_buffer_unmap (&rtcp); + return buf; +} + +typedef struct +{ + GstHarness *h_rtp; + GstHarness *h_rtcp; +} PushRTCPSRCtx; + +static gpointer +_push_sr_buffers (gpointer user_data) +{ + PushRTCPSRCtx *ctx = user_data; + + gst_harness_set_src_caps (ctx->h_rtcp, + gst_caps_new_empty_simple ("application/x-rtcp")); + gst_harness_push (ctx->h_rtcp, generate_rtcp_sr_buffer (1111)); + return NULL; +} + +static gpointer +_push_rtp_buffers (gpointer user_data) +{ + PushRTCPSRCtx *ctx = user_data; + + g_usleep (10); + + gst_harness_set_src_caps (ctx->h_rtp, + gst_caps_new_simple ("application/x-rtp", + "clock-rate", G_TYPE_INT, 8000, "payload", G_TYPE_INT, 100, NULL)); + gst_harness_push (ctx->h_rtp, generate_rtp_buffer (0, 0, 0, 100, 1111)); + return NULL; +} + +GST_START_TEST (test_recv_rtp_and_rtcp_simultaneously) +{ + guint r; + guint repeats = 1000; + if (RUNNING_ON_VALGRIND) + repeats = 2; + + for (r = 0; r < repeats; r++) { + PushRTCPSRCtx ctx; + GThread *t0; + GThread *t1; + + ctx.h_rtcp = gst_harness_new_with_padnames ("rtpbin", + "recv_rtcp_sink_0", NULL); + ctx.h_rtp = gst_harness_new_with_element (ctx.h_rtcp->element, + "recv_rtp_sink_0", NULL); + + t0 = g_thread_new ("push sr", _push_sr_buffers, &ctx); + t1 = g_thread_new ("push rtp", _push_rtp_buffers, &ctx); + + g_thread_join (t0); + g_thread_join (t1); + gst_harness_teardown (ctx.h_rtp); + gst_harness_teardown (ctx.h_rtcp); + } +} + +GST_END_TEST; + static Suite * rtpbin_suite (void) { @@ -987,6 +1088,7 @@ rtpbin_suite (void) tcase_add_test (tc_chain, test_aux_receiver); tcase_add_test (tc_chain, test_sender_eos); tcase_add_test (tc_chain, test_quick_shutdown); + tcase_add_test (tc_chain, test_recv_rtp_and_rtcp_simultaneously); return s; } diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c b/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c index 676127ca18d..fee82794673 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpfunnel.c @@ -128,7 +128,7 @@ GST_START_TEST (rtpfunnel_common_ts_offset) GST_END_TEST; static GstBuffer * -generate_test_buffer (guint seqnum, guint ssrc, guint8 twcc_ext_id) +generate_test_buffer (guint seqnum, guint ssrc) { GstBuffer *buf; GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; @@ -143,13 +143,6 @@ generate_test_buffer (guint seqnum, guint ssrc, guint8 twcc_ext_id) gst_rtp_buffer_set_timestamp (&rtp, seqnum * 160); gst_rtp_buffer_set_ssrc (&rtp, ssrc); - if (twcc_ext_id > 0) { - guint16 data; - GST_WRITE_UINT16_BE (&data, seqnum); - gst_rtp_buffer_add_extension_onebyte_header (&rtp, twcc_ext_id, - &data, sizeof (guint16)); - } - gst_rtp_buffer_unmap (&rtp); return buf; @@ -187,7 +180,7 @@ GST_START_TEST (rtpfunnel_custom_sticky) /* Send a buffer through first pad, expect the event to be the first one */ fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h0, generate_test_buffer (500, 123, 0))); + gst_harness_push (h0, generate_test_buffer (500, 123))); for (;;) { event = gst_harness_pull_event (h); if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_DOWNSTREAM_STICKY) @@ -205,7 +198,7 @@ GST_START_TEST (rtpfunnel_custom_sticky) /* Send a buffer through second pad, expect the event to be the second one */ fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h1, generate_test_buffer (500, 123, 0))); + gst_harness_push (h1, generate_test_buffer (500, 123))); for (;;) { event = gst_harness_pull_event (h); if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_DOWNSTREAM_STICKY) @@ -224,7 +217,7 @@ GST_START_TEST (rtpfunnel_custom_sticky) * one */ fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h0, generate_test_buffer (500, 123, 5))); + gst_harness_push (h0, generate_test_buffer (500, 123))); for (;;) { event = gst_harness_pull_event (h); if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_DOWNSTREAM_STICKY) @@ -296,119 +289,7 @@ GST_START_TEST (rtpfunnel_stress) GST_END_TEST; -#define TWCC_EXTMAP_STR "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01" - -#define BOGUS_EXTMAP_STR "http://www.ietf.org/id/bogus" - -GST_START_TEST (rtpfunnel_twcc_caps) -{ - GstHarness *h, *h0, *h1; - GstCaps *caps, *expected_caps; - - h = gst_harness_new_with_padnames ("rtpfunnel", NULL, "src"); - - /* request a sinkpad, set caps with twcc extmap */ - h0 = gst_harness_new_with_element (h->element, "sink_0", NULL); - gst_harness_set_src_caps_str (h0, "application/x-rtp, " - "ssrc=(uint)123, extmap-5=" TWCC_EXTMAP_STR ""); - - /* request a second sinkpad, the extmap should not be - present in the caps when doing a caps-query downstream, - as we don't want to force upstream (typically a payloader) - to use the extension */ - h1 = gst_harness_new_with_element (h->element, "sink_1", NULL); - caps = gst_pad_query_caps (GST_PAD_PEER (h1->srcpad), NULL); - expected_caps = gst_caps_new_empty_simple ("application/x-rtp"); - fail_unless (gst_caps_is_equal (expected_caps, caps)); - gst_caps_unref (caps); - gst_caps_unref (expected_caps); - - /* now try and set a different extmap for the same id on the other - * sinkpad, and verify this does not work */ - gst_harness_set_src_caps_str (h1, "application/x-rtp, " - "ssrc=(uint)456, extmap-5=" BOGUS_EXTMAP_STR ""); - caps = gst_pad_get_current_caps (GST_PAD_PEER (h1->srcpad)); - fail_unless (caps == NULL); - - /* ...but setting the right extmap (5) will work just fine */ - expected_caps = gst_caps_from_string ("application/x-rtp, " - "ssrc=(uint)456, extmap-5=" TWCC_EXTMAP_STR ""); - gst_harness_set_src_caps (h1, gst_caps_ref (expected_caps)); - caps = gst_pad_get_current_caps (GST_PAD_PEER (h1->srcpad)); - fail_unless (gst_caps_is_equal (expected_caps, caps)); - gst_caps_unref (caps); - gst_caps_unref (expected_caps); - - gst_harness_teardown (h); - gst_harness_teardown (h0); - gst_harness_teardown (h1); -} - -GST_END_TEST; - -static gint32 -get_twcc_seqnum (GstBuffer * buf, guint8 ext_id) -{ - GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; - gint32 val = -1; - gpointer data; - - gst_rtp_buffer_map (buf, GST_MAP_READ, &rtp); - if (gst_rtp_buffer_get_extension_onebyte_header (&rtp, ext_id, - 0, &data, NULL)) { - val = GST_READ_UINT16_BE (data); - } - gst_rtp_buffer_unmap (&rtp); - - return val; -} - -static guint32 -get_ssrc (GstBuffer * buf) -{ - guint32 ssrc; - GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; - gst_rtp_buffer_map (buf, GST_MAP_READ, &rtp); - ssrc = gst_rtp_buffer_get_ssrc (&rtp); - gst_rtp_buffer_unmap (&rtp); - return ssrc; -} - -GST_START_TEST (rtpfunnel_twcc_passthrough) -{ - GstHarness *h, *h0; - GstBuffer *buf; - guint16 offset = 65530; - guint packets = 40; - guint i; - - h = gst_harness_new_with_padnames ("rtpfunnel", NULL, "src"); - h0 = gst_harness_new_with_element (h->element, "sink_0", NULL); - gst_harness_set_src_caps_str (h0, "application/x-rtp, " - "ssrc=(uint)123, extmap-5=" TWCC_EXTMAP_STR ""); - - /* push some packets with twcc seqnum */ - for (i = 0; i < packets; i++) { - guint16 seqnum = i + offset; - fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h0, generate_test_buffer (seqnum, 123, 5))); - } - - /* and verify the seqnums stays unchanged through the funnel */ - for (i = 0; i < packets; i++) { - guint16 seqnum = i + offset; - buf = gst_harness_pull (h); - fail_unless_equals_int (seqnum, get_twcc_seqnum (buf, 5)); - gst_buffer_unref (buf); - } - - gst_harness_teardown (h); - gst_harness_teardown (h0); -} - -GST_END_TEST; - -GST_START_TEST (rtpfunnel_twcc_mux) +GST_START_TEST (rtpfunnel_mark_media_buffer_flags) { GstHarness *h, *h0, *h1; GstBuffer *buf; @@ -417,25 +298,21 @@ GST_START_TEST (rtpfunnel_twcc_mux) h0 = gst_harness_new_with_element (h->element, "sink_0", NULL); h1 = gst_harness_new_with_element (h->element, "sink_1", NULL); gst_harness_set_src_caps_str (h0, "application/x-rtp, " - "ssrc=(uint)123, extmap-5=" TWCC_EXTMAP_STR ""); + "ssrc=(uint)123, media=(string)audio"); gst_harness_set_src_caps_str (h1, "application/x-rtp, " - "ssrc=(uint)456, extmap-5=" TWCC_EXTMAP_STR ""); + "ssrc=(uint)123, media=(string)video"); - /* push buffers on both pads with different twcc-seqnums on them (500 and 60000) */ fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h0, generate_test_buffer (500, 123, 5))); + gst_harness_push (h0, generate_test_buffer (0, 123))); fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h1, generate_test_buffer (60000, 321, 5))); + gst_harness_push (h1, generate_test_buffer (0, 321))); - /* verify they are muxed continuously (0 -> 1) */ buf = gst_harness_pull (h); - fail_unless_equals_int (123, get_ssrc (buf)); - fail_unless_equals_int (0, get_twcc_seqnum (buf, 5)); + fail_unless (GST_BUFFER_FLAG_IS_SET (buf, GST_RTP_BUFFER_FLAG_MEDIA_AUDIO)); gst_buffer_unref (buf); buf = gst_harness_pull (h); - fail_unless_equals_int (321, get_ssrc (buf)); - fail_unless_equals_int (1, get_twcc_seqnum (buf, 5)); + fail_unless (GST_BUFFER_FLAG_IS_SET (buf, GST_RTP_BUFFER_FLAG_MEDIA_VIDEO)); gst_buffer_unref (buf); gst_harness_teardown (h); @@ -445,44 +322,42 @@ GST_START_TEST (rtpfunnel_twcc_mux) GST_END_TEST; - -GST_START_TEST (rtpfunnel_twcc_passthrough_then_mux) +GST_START_TEST (rtpfunnel_terminate_latency) { GstHarness *h, *h0, *h1; GstBuffer *buf; - guint offset0 = 500; - guint offset1 = 45678; - guint i; + GstClockTime latency; h = gst_harness_new_with_padnames ("rtpfunnel", NULL, "src"); h0 = gst_harness_new_with_element (h->element, "sink_0", NULL); + h1 = gst_harness_new_with_element (h->element, "sink_1", NULL); gst_harness_set_src_caps_str (h0, "application/x-rtp, " - "ssrc=(uint)123, extmap-5=" TWCC_EXTMAP_STR ""); + "ssrc=(uint)123, media=(string)audio"); + gst_harness_set_src_caps_str (h1, "application/x-rtp, " + "ssrc=(uint)321, media=(string)video"); - /* push one packet with twcc seqnum 100 on pad0 */ - fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h0, generate_test_buffer (offset0, 123, 5))); + gst_harness_set_upstream_latency (h0, 11 * GST_MSECOND); + gst_harness_set_upstream_latency (h1, 12 * GST_MSECOND); - /* add pad1 to the funnel, also with twcc */ - h1 = gst_harness_new_with_element (h->element, "sink_1", NULL); - gst_harness_set_src_caps_str (h1, "application/x-rtp, " - "ssrc=(uint)456, extmap-5=" TWCC_EXTMAP_STR ""); + /* ask funnel about its latency */ + latency = gst_harness_query_latency (h); + /* verify it reports 0 */ + fail_unless_equals_uint64 (latency, 0); - /* push one buffer on both pads, with pad1 starting at a different - offset */ + /* push buffers with PTS = 0 into the funnel */ fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h0, generate_test_buffer (offset0 + 1, 123, 5))); + gst_harness_push (h0, generate_test_buffer (0, 123))); fail_unless_equals_int (GST_FLOW_OK, - gst_harness_push (h1, generate_test_buffer (offset1, 321, 5))); - - /* and verify the seqnums are continuous for all 3 packets, using - the inital offset from pad0 */ - for (i = 0; i < 3; i++) { - guint16 seqnum = i + offset0; - buf = gst_harness_pull (h); - fail_unless_equals_int (seqnum, get_twcc_seqnum (buf, 5)); - gst_buffer_unref (buf); - } + gst_harness_push (h1, generate_test_buffer (0, 321))); + + /* and verify the upstream latency is now reflected as PTS on those + buffers */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (GST_BUFFER_PTS (buf), 11 * GST_MSECOND); + gst_buffer_unref (buf); + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (GST_BUFFER_PTS (buf), 12 * GST_MSECOND); + gst_buffer_unref (buf); gst_harness_teardown (h); gst_harness_teardown (h0); @@ -506,10 +381,8 @@ rtpfunnel_suite (void) tcase_add_test (tc_chain, rtpfunnel_stress); - tcase_add_test (tc_chain, rtpfunnel_twcc_caps); - tcase_add_test (tc_chain, rtpfunnel_twcc_passthrough); - tcase_add_test (tc_chain, rtpfunnel_twcc_mux); - tcase_add_test (tc_chain, rtpfunnel_twcc_passthrough_then_mux); + tcase_add_test (tc_chain, rtpfunnel_mark_media_buffer_flags); + tcase_add_test (tc_chain, rtpfunnel_terminate_latency); return s; } diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtph263.c b/subprojects/gst-plugins-good/tests/check/elements/rtph263.c index 28c01981fe2..afdba783d70 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtph263.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtph263.c @@ -133,8 +133,7 @@ GST_START_TEST (test_h263pay_mode_b_snow) if (!have_element ("avenc_h263")) return; - h = gst_harness_new_parse - ("avenc_h263 rtp-payload-size=1 ! rtph263pay mtu=1350 "); + h = gst_harness_new_parse ("avenc_h263 ps=1 ! rtph263pay mtu=1350 "); gst_harness_add_src_parse (h, "videotestsrc pattern=snow is-live=1 ! " "capsfilter caps=\"video/x-raw,format=I420,width=176,height=144\"", TRUE); diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtphdrext-roi.c b/subprojects/gst-plugins-good/tests/check/elements/rtphdrext-roi.c new file mode 100644 index 00000000000..a975b4f27b7 --- /dev/null +++ b/subprojects/gst-plugins-good/tests/check/elements/rtphdrext-roi.c @@ -0,0 +1,1005 @@ +/* GStreamer + * + * unit test for rtphdrext-roi elements + * + * Copyright (C) 2020-2021 Pexip AS. + * @author: Camilo Celis Guzman + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include +#include +#include +#include + +#define URI GST_RTP_HDREXT_BASE "TBD:draft-ford-avtcore-roi-extension-00" +#define EXTMAP_ID 10 + +#define SRC_CAPS_STR "video/x-raw,format=I420" + +#define MAX_ROI_TYPES_ALLOWED 15 + +GST_START_TEST (test_rtphdrext_roi_basic) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = gst_caps_from_string (SRC_CAPS_STR); + gst_harness_set_src_caps (h, src_caps); + + gst_harness_add_src_parse (h, "videotestsrc ! " + "capsfilter caps=\"" SRC_CAPS_STR "\"", FALSE); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + /* verify that we can push and pull buffers */ + gst_harness_play (h); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push_from_src (h)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_harness_teardown (h); +} + +GST_END_TEST; + +static GstCaps * +_i420_caps (gint width, gint height) +{ + GstVideoInfo info; + gst_video_info_init (&info); + gst_video_info_set_format (&info, GST_VIDEO_FORMAT_I420, width, height); + GST_VIDEO_INFO_FPS_N (&info) = 1; + GST_VIDEO_INFO_FPS_D (&info) = 1; + GST_VIDEO_INFO_PAR_N (&info) = 1; + GST_VIDEO_INFO_PAR_D (&info) = 1; + return gst_video_info_to_caps (&info); +} + +static GstBuffer * +_gst_harness_create_raw_buffer (GstHarness * h, GstCaps * caps) +{ + GstBuffer *buf; + GstVideoInfo video_info; + + gst_video_info_init (&video_info); + gst_video_info_from_caps (&video_info, caps); + + buf = gst_harness_create_buffer (h, GST_VIDEO_INFO_SIZE (&video_info)); + gst_buffer_memset (buf, 0, 0, GST_VIDEO_INFO_SIZE (&video_info)); + fail_unless (buf); + + gst_buffer_add_video_meta_full (buf, + GST_VIDEO_FRAME_FLAG_NONE, + GST_VIDEO_INFO_FORMAT (&video_info), + GST_VIDEO_INFO_WIDTH (&video_info), + GST_VIDEO_INFO_HEIGHT (&video_info), + GST_VIDEO_INFO_N_PLANES (&video_info), + video_info.offset, video_info.stride); + + GST_BUFFER_PTS (buf) = 0; + GST_BUFFER_DURATION (buf) = GST_SECOND / 30; + + return buf; +} + +GST_START_TEST (test_rtphdrext_roi_default_roi_type) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + + GstMeta *meta; + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + const GQuark default_roi_type = 126; + + gpointer state = NULL; + gint num_roi_metas_found = 0; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = _i420_caps (640, 360); + gst_harness_set_src_caps (h, gst_caps_ref (src_caps)); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + /* verify that we can push and pull buffers */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + /* add a RoI meta on an input buffer with the default RoI type + * and expect this to be payloaded and depayloaded back as a RoI meta on + * the output buffer */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + gst_buffer_add_video_region_of_interest_meta_id (buf, + default_roi_type, 0, 0, 640, 360); + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify that we get exactly one RoI meta on the output buffer with + * the expected RoI coordinates and type */ + while ((meta = gst_buffer_iterate_meta (buf, &state))) { + if (meta->info->api == meta_info->api) { + GstVideoRegionOfInterestMeta *vmeta = + (GstVideoRegionOfInterestMeta *) meta; + fail_unless_equals_int ((gint) default_roi_type, (gint) vmeta->roi_type); + fail_unless_equals_int (0, vmeta->x); + fail_unless_equals_int (0, vmeta->y); + fail_unless_equals_int (640, vmeta->w); + fail_unless_equals_int (360, vmeta->h); + num_roi_metas_found++; + } + } + fail_unless_equals_int (1, num_roi_metas_found); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_caps_unref (src_caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtphdrext_roi_unknown_roi_type) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + + GQuark other_roi_type = g_quark_from_string ("SomeRoITypeStr"); + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = _i420_caps (640, 360); + gst_harness_set_src_caps (h, gst_caps_ref (src_caps)); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + /* verify that we can push and pull buffers */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + /* add a RoI meta on an input buffer with an unknown RoI type and + * don't expect this to be payloaded and depayloaded as a RoI meta on + * the output buffer as we strictly only payload known RoI types */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + gst_buffer_add_video_region_of_interest_meta_id (buf, + other_roi_type, 0, 0, 640, 360); + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_caps_unref (src_caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtphdrext_roi_explicit_roi_type) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + + GstMeta *meta; + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + GValue roi_types = G_VALUE_INIT, roi_type = G_VALUE_INIT; + const GQuark other_roi_type = 12345; + + gpointer state = NULL; + gint num_roi_metas_found = 0; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = _i420_caps (640, 360); + gst_harness_set_src_caps (h, gst_caps_ref (src_caps)); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + gst_value_array_init (&roi_types, 1); + g_value_init (&roi_type, G_TYPE_UINT); + g_value_set_uint (&roi_type, other_roi_type); + gst_value_array_append_value (&roi_types, &roi_type); + + /* explicitly set a roi-type on both payloader and depayloader extensions + * and expect a buffer with such roi-type added to their RoI meta to be + * correctly payloaded and depayloaded */ + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + g_object_set_property ((GObject *) pay_ext, "roi-types", &roi_types); + g_object_set_property ((GObject *) depay_ext, "roi-types", &roi_types); + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + /* verify that we can push and pull buffers */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + gst_buffer_unref (gst_harness_pull (h)); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + buf = _gst_harness_create_raw_buffer (h, src_caps); + gst_buffer_add_video_region_of_interest_meta_id (buf, + other_roi_type, 0, 0, 640, 360); + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify that we get exactly one RoI meta on the output buffer with + * the expected RoI coordinates and type */ + while ((meta = gst_buffer_iterate_meta (buf, &state))) { + if (meta->info->api == meta_info->api) { + GstVideoRegionOfInterestMeta *vmeta = + (GstVideoRegionOfInterestMeta *) meta; + fail_unless_equals_int ((gint) other_roi_type, (gint) vmeta->roi_type); + fail_unless_equals_int (0, vmeta->x); + fail_unless_equals_int (0, vmeta->y); + fail_unless_equals_int (640, vmeta->w); + fail_unless_equals_int (360, vmeta->h); + num_roi_metas_found++; + } + } + fail_unless_equals_int (1, num_roi_metas_found); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_caps_unref (src_caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtphdrext_roi_explicit_roi_type_pay_only) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + GValue roi_types = G_VALUE_INIT, roi_type = G_VALUE_INIT; + + const GQuark default_roi_type = 126; + const GQuark other_roi_type = default_roi_type / 2; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = _i420_caps (640, 360); + gst_harness_set_src_caps (h, gst_caps_ref (src_caps)); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + /* set a roi-type on the payloader only and expect this to NOT be + * depayloaded as the depayloader will only depayload data using a specific + * set of specified roi-types */ + gst_value_array_init (&roi_types, 1); + g_value_init (&roi_type, G_TYPE_UINT); + g_value_set_uint (&roi_type, other_roi_type); + gst_value_array_append_value (&roi_types, &roi_type); + + g_object_set_property (G_OBJECT (pay_ext), "roi-types", &roi_types); + + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + /* verify that we can push and pull buffers */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + buf = _gst_harness_create_raw_buffer (h, src_caps); + gst_buffer_add_video_region_of_interest_meta_id (buf, + other_roi_type, 0, 0, 640, 360); + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_caps_unref (src_caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtphdrext_roi_explicit_roi_type_depay_only) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + GValue roi_types = G_VALUE_INIT, roi_type = G_VALUE_INIT; + + const GQuark default_roi_type = 126; + const GQuark other_roi_type = default_roi_type / 2; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = _i420_caps (640, 360); + gst_harness_set_src_caps (h, gst_caps_ref (src_caps)); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + /* set a roi-type on the depayloader and expect nothing to be depayloaded + * nor payloaded as even if a buffer has RoI meta we would have not signal + * the rigt roi-type to payload, hence no header extensions would be added */ + gst_value_array_init (&roi_types, 1); + g_value_init (&roi_type, G_TYPE_UINT); + g_value_set_uint (&roi_type, other_roi_type); + gst_value_array_append_value (&roi_types, &roi_type); + + g_object_set_property (G_OBJECT (depay_ext), "roi-types", &roi_types); + + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + /* verify that we can push and pull buffers */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + buf = _gst_harness_create_raw_buffer (h, src_caps); + gst_buffer_add_video_region_of_interest_meta_id (buf, + other_roi_type, 0, 0, 640, 360); + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify that we get no metas as we would not be able to payload an + * unknown type hence no RTP header extension is added and the depayloader + * wont, ofcourse add any RoI metas on the output buffer */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_caps_unref (src_caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtphdrext_roi_multiple_metas_with_same_roi_type) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + + GstMeta *meta; + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + const GQuark default_roi_type = 126; + + gpointer state = NULL; + gint num_roi_metas_found = 0; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = _i420_caps (640, 360); + gst_harness_set_src_caps (h, gst_caps_ref (src_caps)); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + /* verify that we can push and pull buffers */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + /* add multiple RoI meta on an input buffer all with the same default RoI type + * and expect only the first one to be be payloaded and depayloaded back as + * a RoI meta on the output buffer */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + gst_buffer_add_video_region_of_interest_meta_id (buf, + default_roi_type, 0, 0, 640, 360); + gst_buffer_add_video_region_of_interest_meta_id (buf, + default_roi_type, 1, 1, 640, 360); + gst_buffer_add_video_region_of_interest_meta_id (buf, + default_roi_type, 2, 2, 640, 360); + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify that we get exactly one RoI meta on the output buffer with + * the expected RoI coordinates and type */ + while ((meta = gst_buffer_iterate_meta (buf, &state))) { + if (meta->info->api == meta_info->api) { + GstVideoRegionOfInterestMeta *vmeta = + (GstVideoRegionOfInterestMeta *) meta; + fail_unless_equals_int ((gint) default_roi_type, (gint) vmeta->roi_type); + fail_unless_equals_int (0, vmeta->x); + fail_unless_equals_int (0, vmeta->y); + fail_unless_equals_int (640, vmeta->w); + fail_unless_equals_int (360, vmeta->h); + num_roi_metas_found++; + } + } + fail_unless_equals_int (1, num_roi_metas_found); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_caps_unref (src_caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtphdrext_roi_multiple_roi_types) +{ + GstHarness *h; + GstCaps *src_caps, *pay_pad_caps, *expected_caps; + + GstElement *pay, *depay; + GstRTPHeaderExtension *pay_ext, *depay_ext; + GstPad *pay_pad; + + GstBuffer *buf; + + GstMeta *meta; + const GstMetaInfo *meta_info = GST_VIDEO_REGION_OF_INTEREST_META_INFO; + + GValue roi_types = G_VALUE_INIT, roi_type = G_VALUE_INIT; + const GQuark roi_type_base = 0xDEADBEEF; + + gpointer state = NULL; + gint num_roi_metas_found = 0; + guint i = 0; + + h = gst_harness_new_parse ("rtpvrawpay ! rtpvrawdepay"); + + src_caps = _i420_caps (640, 360); + gst_harness_set_src_caps (h, gst_caps_ref (src_caps)); + + pay = gst_harness_find_element (h, "rtpvrawpay"); + depay = gst_harness_find_element (h, "rtpvrawdepay"); + + pay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + depay_ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + /* allow all roi-types from 1 to MAX_ROI_TYPES_ALLOWED on both the depay and pay ext */ + gst_value_array_init (&roi_types, MAX_ROI_TYPES_ALLOWED); + g_value_init (&roi_type, G_TYPE_UINT); + for (i = 0; i < MAX_ROI_TYPES_ALLOWED; i++) { + g_value_set_uint (&roi_type, roi_type_base + i); + gst_value_array_append_value (&roi_types, &roi_type); + } + g_object_set_property ((GObject *) pay_ext, "roi-types", &roi_types); + g_object_set_property ((GObject *) depay_ext, "roi-types", &roi_types); + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + gst_rtp_header_extension_set_id (pay_ext, EXTMAP_ID); + gst_rtp_header_extension_set_id (depay_ext, EXTMAP_ID); + + g_signal_emit_by_name (pay, "add-extension", pay_ext); + g_signal_emit_by_name (depay, "add-extension", depay_ext); + + /* verify that we can push and pull buffers */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify the presence of RoI URI on the depayloader caps */ + pay_pad = gst_element_get_static_pad (pay, "src"); + pay_pad_caps = gst_pad_get_current_caps (pay_pad); + expected_caps = + gst_caps_from_string ("application/x-rtp, extmap-" G_STRINGIFY (EXTMAP_ID) + "=" URI); + fail_unless (gst_caps_is_subset (pay_pad_caps, expected_caps)); + gst_object_unref (pay_pad); + gst_caps_unref (pay_pad_caps); + gst_caps_unref (expected_caps); + + /* verify there are NO RoI meta on the buffer pulled from the depayloader */ + fail_if (gst_buffer_get_meta (buf, meta_info->api)); + gst_buffer_unref (buf); + + /* add a RoI meta for each one of the allowed roi-types an the input buffer + * and expect all to be payloaded and depayloaded back as a RoI metas on + * the output buffer */ + buf = _gst_harness_create_raw_buffer (h, src_caps); + for (i = 0; i < MAX_ROI_TYPES_ALLOWED; i++) { + /* re-use x,y coordinates to make all RoI coordinates different */ + gst_buffer_add_video_region_of_interest_meta_id (buf, + roi_type_base + i, i, i, 640, 360); + } + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify that we get exactly MAX_ROI_TYPES_ALLOWED RoI metas on the output buffer with + * the expected RoI coordinates and type */ + i = 0; + while ((meta = gst_buffer_iterate_meta (buf, &state))) { + if (meta->info->api == meta_info->api) { + GstVideoRegionOfInterestMeta *vmeta = + (GstVideoRegionOfInterestMeta *) meta; + fail_unless_equals_int (roi_type_base + i, (gint) vmeta->roi_type); + fail_unless_equals_int (i, vmeta->x); + fail_unless_equals_int (i, vmeta->y); + fail_unless_equals_int (640, vmeta->w); + fail_unless_equals_int (360, vmeta->h); + num_roi_metas_found++; + i++; + } + } + fail_unless_equals_int (MAX_ROI_TYPES_ALLOWED, num_roi_metas_found); + gst_buffer_unref (buf); + + /* now simply add half as many RoI metas and expect them all to be paylaoded + * and depayloaded */ + num_roi_metas_found = 0; + buf = _gst_harness_create_raw_buffer (h, src_caps); + for (i = 0; i < MAX_ROI_TYPES_ALLOWED / 2; i++) { + /* re-use x,y coordinates to make all RoI coordinates different */ + gst_buffer_add_video_region_of_interest_meta_id (buf, + roi_type_base + i, i, i, 640, 360); + } + + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + buf = gst_harness_pull (h); + fail_unless (buf); + + /* verify that we get exactly MAX_ROI_TYPES_ALLOWED RoI metas on the output buffer with + * the expected RoI coordinates and type */ + i = 0; + while ((meta = gst_buffer_iterate_meta (buf, &state))) { + if (meta->info->api == meta_info->api) { + GstVideoRegionOfInterestMeta *vmeta = + (GstVideoRegionOfInterestMeta *) meta; + fail_unless_equals_int (roi_type_base + i, (gint) vmeta->roi_type); + fail_unless_equals_int (i, vmeta->x); + fail_unless_equals_int (i, vmeta->y); + fail_unless_equals_int (640, vmeta->w); + fail_unless_equals_int (360, vmeta->h); + num_roi_metas_found++; + i++; + } + } + fail_unless_equals_int (MAX_ROI_TYPES_ALLOWED / 2, num_roi_metas_found); + gst_buffer_unref (buf); + + gst_object_unref (pay_ext); + gst_object_unref (depay_ext); + gst_object_unref (pay); + gst_object_unref (depay); + gst_caps_unref (src_caps); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtphdrext_roi_property_roi_types) +{ + GstRTPHeaderExtension *ext; + GValue roi_types = G_VALUE_INIT, roi_type = G_VALUE_INIT; + GValue invalid_roi_type = G_VALUE_INIT; + const GValue *val_ref; + const GQuark default_roi_type = 126; + gboolean seen_roi_type[23] = { 0 }; + + ext = + GST_RTP_HEADER_EXTENSION (gst_element_factory_make ("rtphdrextroi", + NULL)); + + /* Case 1: default roi-types: + * roi-types must be a GValue array of length 1 with a 0 roi-type */ + gst_value_array_init (&roi_types, 1); + g_object_get_property ((GObject *) ext, "roi-types", &roi_types); + fail_unless_equals_int (1, (gint) gst_value_array_get_size (&roi_types)); + val_ref = gst_value_array_get_value (&roi_types, 0); + fail_unless (G_VALUE_HOLDS_UINT (val_ref)); + fail_unless_equals_int (default_roi_type, g_value_get_uint (val_ref)); + g_value_unset (&roi_types); + + /* Case 2: set a valid GValue array with only uints */ + gst_value_array_init (&roi_types, MAX_ROI_TYPES_ALLOWED); + g_value_init (&roi_type, G_TYPE_UINT); + for (guint i = 1; i <= MAX_ROI_TYPES_ALLOWED; i++) { + g_value_set_uint (&roi_type, i); + gst_value_array_append_value (&roi_types, &roi_type); + } + g_object_set_property ((GObject *) ext, "roi-types", &roi_types); + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + g_object_get_property ((GObject *) ext, "roi-types", &roi_types); + fail_unless_equals_int (MAX_ROI_TYPES_ALLOWED, + gst_value_array_get_size (&roi_types)); + for (guint i = 0; i < MAX_ROI_TYPES_ALLOWED; i++) { + const GValue *v = gst_value_array_get_value (&roi_types, i); + fail_unless (G_VALUE_HOLDS_UINT (v)); + fail_if (seen_roi_type[g_value_get_uint (v)]); + seen_roi_type[g_value_get_uint (v)] = TRUE; + } + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + /* Case 3: invalid roi-type within GValue array */ + gst_value_array_init (&roi_types, 1); + g_value_init (&roi_type, G_TYPE_UINT); + /* 0 is forbiden as a roi-type */ + g_value_set_uint (&roi_type, 0); + gst_value_array_append_value (&roi_types, &roi_type); + + // FIXME: ASSERT_WARNING broken + // ASSERT_WARNING (g_object_set_property ((GObject *) ext, "roi-types", + // &roi_types)); + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + /* Case 4: invalid roi-type within GValue array */ + gst_value_array_init (&roi_types, 1); + g_value_init (&invalid_roi_type, G_TYPE_STRING); + g_value_set_string (&invalid_roi_type, "Dummy"); + gst_value_array_append_value (&roi_types, &invalid_roi_type); + + // FIXME: ASSERT_WARNING broken + // ASSERT_WARNING (g_object_set_property ((GObject *) ext, "roi-types", + // &roi_types)); + g_value_unset (&invalid_roi_type); + g_value_unset (&roi_types); + + /* Case 5: duplicate roi-types on GValue array */ + gst_value_array_init (&roi_types, 3); + g_value_init (&roi_type, G_TYPE_UINT); + /* valid */ + g_value_set_uint (&roi_type, 1); + gst_value_array_append_value (&roi_types, &roi_type); + /* valid */ + g_value_set_uint (&roi_type, 2); + gst_value_array_append_value (&roi_types, &roi_type); + /* duplicate */ + g_value_set_uint (&roi_type, 2); + gst_value_array_append_value (&roi_types, &roi_type); + + // FIXME: ASSERT_WARNING broken + // ASSERT_WARNING (g_object_set_property ((GObject *) ext, "roi-types", + // &roi_types)); + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + /* Case 5: invalid length */ + gst_value_array_init (&roi_types, 500); + g_value_init (&roi_type, G_TYPE_UINT); + for (guint i = 0; i < 500; i++) { + g_value_set_uint (&roi_type, i); + gst_value_array_append_value (&roi_types, &roi_type); + } + + // FIXME: ASSERT_WARNING broken + // ASSERT_WARNING (g_object_set_property ((GObject *) ext, "roi-types", + // &roi_types)); + g_value_unset (&roi_type); + g_value_unset (&roi_types); + + gst_object_unref (ext); +} + +GST_END_TEST; + +static Suite * +rtphdrext_roi_suite (void) +{ + Suite *s = suite_create ("rtphdrext_roi"); + TCase *tc_chain = tcase_create ("general"); + + suite_add_tcase (s, tc_chain); + + tcase_add_test (tc_chain, test_rtphdrext_roi_basic); + tcase_add_test (tc_chain, test_rtphdrext_roi_default_roi_type); + tcase_add_test (tc_chain, test_rtphdrext_roi_unknown_roi_type); + tcase_add_test (tc_chain, test_rtphdrext_roi_explicit_roi_type); + tcase_add_test (tc_chain, test_rtphdrext_roi_explicit_roi_type_pay_only); + tcase_add_test (tc_chain, test_rtphdrext_roi_explicit_roi_type_depay_only); + + tcase_add_test (tc_chain, + test_rtphdrext_roi_multiple_metas_with_same_roi_type); + tcase_add_test (tc_chain, test_rtphdrext_roi_multiple_roi_types); + + tcase_add_test (tc_chain, test_rtphdrext_roi_property_roi_types); + + return s; +} + +GST_CHECK_MAIN (rtphdrext_roi) diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c index 8984c350818..a6122e45bbd 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpjitterbuffer.c @@ -159,12 +159,19 @@ start_jitterbuffer (GstElement * jitterbuffer) return ret; } +/* wrapper for glib to get correct function signature */ +static void +gst_mini_object_unref_func (gpointer data, G_GNUC_UNUSED gpointer user_data) +{ + gst_mini_object_unref (data); +} + static void cleanup_jitterbuffer (GstElement * jitterbuffer) { GST_DEBUG ("cleanup_jitterbuffer"); - g_list_foreach (buffers, (GFunc) gst_mini_object_unref, NULL); + g_list_foreach (buffers, gst_mini_object_unref_func, NULL); g_list_free (buffers); buffers = NULL; @@ -650,15 +657,16 @@ verify_jb_stats (GstElement * jb, GstStructure * expected) } static guint -construct_deterministic_initial_state (GstHarness * h, gint latency_ms) +construct_deterministic_initial_state_full (GstHarness * h, + guint start_seqnum, gint latency_ms, GstCaps * caps) { - guint next_seqnum = latency_ms / TEST_BUF_MS + 1; + guint next_seqnum = start_seqnum + (latency_ms / TEST_BUF_MS + 1); guint seqnum; gint i; g_assert (latency_ms % TEST_BUF_MS == 0); - gst_harness_set_src_caps (h, generate_caps ()); + gst_harness_set_src_caps (h, caps); g_object_set (h->element, "latency", latency_ms, NULL); /* When the first packet arrives in the jitterbuffer, it will create a @@ -681,7 +689,7 @@ construct_deterministic_initial_state (GstHarness * h, gint latency_ms) */ /* Packet 0 arrives at time 0ms, Packet 5 arrives at time 100ms */ - for (seqnum = 0; seqnum < next_seqnum; seqnum++) { + for (seqnum = start_seqnum; seqnum < next_seqnum; seqnum++) { push_test_buffer (h, seqnum); gst_harness_wait_for_clock_id_waits (h, 1, 60); } @@ -692,8 +700,9 @@ construct_deterministic_initial_state (GstHarness * h, gint latency_ms) */ gst_harness_crank_single_clock_wait (h); fail_unless_equals_int64 (latency_ms * GST_MSECOND, - gst_clock_get_time (GST_ELEMENT_CLOCK (h->element))); - for (seqnum = 0; seqnum < next_seqnum; seqnum++) { + gst_clock_get_time (GST_ELEMENT_CLOCK (h->element)) - + (start_seqnum * TEST_BUF_MS * GST_MSECOND)); + for (seqnum = start_seqnum; seqnum < next_seqnum; seqnum++) { GstBuffer *buf = gst_harness_pull (h); fail_unless_equals_uint64 (seqnum * TEST_BUF_DURATION, GST_BUFFER_PTS (buf)); @@ -705,6 +714,10 @@ construct_deterministic_initial_state (GstHarness * h, gint latency_ms) for (i = 0; i < 3; i++) gst_event_unref (gst_harness_pull_event (h)); + /* pull out the latency-changed event */ + if (gst_harness_events_in_queue (h) == 1) + gst_event_unref (gst_harness_pull_event (h)); + /* drop reconfigure event */ gst_event_unref (gst_harness_pull_upstream_event (h)); @@ -780,6 +793,13 @@ setup_rtcp_pads (GstElement * jitterbuffer) return rtcp_fxsrc_pad; } +static guint +construct_deterministic_initial_state (GstHarness * h, gint latency_ms) +{ + return construct_deterministic_initial_state_full (h, 0, latency_ms, + generate_caps ()); +} + GST_START_TEST (test_lost_event) { GstHarness *h = gst_harness_new ("rtpjitterbuffer"); @@ -1242,6 +1262,9 @@ GST_START_TEST (test_all_packets_are_timestamped_zero) gst_harness_set_src_caps (h, generate_caps ()); g_object_set (h->element, "do-lost", TRUE, "latency", jb_latency_ms, NULL); + /* pull out the latency-changed event */ + gst_event_unref (gst_harness_pull_event (h)); + /* advance the clock with 10 seconds */ gst_harness_set_time (h, 10 * GST_SECOND); @@ -1361,6 +1384,9 @@ GST_START_TEST (test_loss_equidistant_spacing_with_parameter_packets) gst_harness_set_src_caps (h, generate_caps ()); g_object_set (h->element, "do-lost", TRUE, "latency", latency_ms, NULL); + /* pull out the latency-changed event */ + gst_event_unref (gst_harness_pull_event (h)); + /* drop stream-start, caps, segment */ for (i = 0; i < 3; i++) gst_event_unref (gst_harness_pull_event (h)); @@ -2381,10 +2407,9 @@ GST_START_TEST (test_rtx_with_backwards_rtptime) * we need to advance the clock to the expected point */ gst_harness_wait_for_clock_id_waits (h, 1, 1); - gst_harness_set_time (h, 6 * TEST_BUF_DURATION + 15 * GST_MSECOND); + gst_harness_set_time (h, 6 * TEST_BUF_DURATION); gst_harness_crank_single_clock_wait (h); - verify_rtx_event (h, 6, 5 * TEST_BUF_DURATION + 15 * GST_MSECOND, - 20, 35 * GST_MSECOND); + verify_rtx_event (h, 6, 5 * TEST_BUF_DURATION, 20, 20 * GST_MSECOND); fail_unless (verify_jb_stats (h->element, gst_structure_new ("application/x-rtp-jitterbuffer-stats", @@ -2463,8 +2488,8 @@ start_test_rtx_large_packet_spacing (GstHarness * h, gst_buffer_unref (gst_harness_pull (h)); } - /* drop GstEventStreamStart & GstEventCaps & GstEventSegment */ - for (i = 0; i < 3; i++) + /* drop GstEventStreamStart & GstEventCaps & GstEventSegment & Latency */ + for (i = 0; i < 4; i++) gst_event_unref (gst_harness_pull_event (h)); /* drop reconfigure event */ gst_event_unref (gst_harness_pull_upstream_event (h)); @@ -2599,8 +2624,8 @@ GST_START_TEST (test_rtx_large_packet_spacing_does_not_reset_jitterbuffer) gst_buffer_unref (buffer); } - /* drop GstEventStreamStart & GstEventCaps & GstEventSegment */ - for (i = 0; i < 3; i++) + /* drop GstEventStreamStart & GstEventCaps & GstEventSegment & Latency */ + for (i = 0; i < 4; i++) gst_event_unref (gst_harness_pull_event (h)); /* drop reconfigure event */ gst_event_unref (gst_harness_pull_upstream_event (h)); @@ -2810,26 +2835,43 @@ GST_START_TEST (test_deadline_ts_offset_overflow) GST_END_TEST; +typedef struct +{ + GstClockTime delta_dts; + guint delta_seqnum; + guint delta_rtptime; +} BigGapCtx; + +/* TEST_RTP_TS_DURATION = 160 */ +static BigGapCtx big_gap_testdata[] = { + {0, 20000, 20000 * 160}, + {0, 10000, 10000 * 160}, + {0, 10000, 0 * 160}, + {0, 3000, 3000 * 160}, +}; + GST_START_TEST (test_big_gap_seqnum) { GstHarness *h = gst_harness_new ("rtpjitterbuffer"); const gint num_consecutive = 5; - const guint gap = 20000; gint i; guint seqnum_org; GstClockTime dts_base; guint seqnum_base; guint32 rtpts_base; GstClockTime expected_ts; + BigGapCtx *ctx = &big_gap_testdata[__i__]; g_object_set (h->element, "do-lost", TRUE, "do-retransmission", TRUE, NULL); seqnum_org = construct_deterministic_initial_state (h, 100); - /* a sudden jump in sequence-numbers (and rtptime), but packets keep arriving - at the same pace */ dts_base = seqnum_org * TEST_BUF_DURATION; - seqnum_base = seqnum_org + gap; - rtpts_base = seqnum_base * TEST_RTP_TS_DURATION; + seqnum_base = seqnum_org; + rtpts_base = seqnum_org * TEST_RTP_TS_DURATION; + + dts_base += ctx->delta_dts; + seqnum_base += ctx->delta_seqnum; + rtpts_base += ctx->delta_rtptime; for (i = 0; i < num_consecutive; i++) { fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, @@ -2897,6 +2939,78 @@ GST_START_TEST (test_big_gap_arrival_time) GST_END_TEST; +static void +test_large_packet_spacing_event_pkt_pts (gboolean do_rtx) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + gint i; + guint seqnum_org; + GstClockTime dts_base; + guint seqnum_base, expected_seqnum; + guint32 rtpts_base; + GstClockTime expected_ts; + const gint num_consecutive = 3; + + g_object_set (h->element, "do-lost", TRUE, "do-retransmission", do_rtx, NULL); + seqnum_org = construct_deterministic_initial_state (h, 100); + + /* packets start coming after 10 seconds */ + dts_base = seqnum_org * 10 * GST_SECOND; + seqnum_base = seqnum_org; + rtpts_base = seqnum_base * TEST_RTP_TS_DURATION; + + for (i = 0; i < num_consecutive; i++) { + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, + generate_test_buffer_full (dts_base + i * TEST_BUF_DURATION, + seqnum_base + i, rtpts_base + i * TEST_RTP_TS_DURATION))); + } + + for (i = 0; i < num_consecutive; i++) { + GstBuffer *buf = gst_harness_pull (h); + guint expected_seqnum = seqnum_base + i; + fail_unless_equals_int (expected_seqnum, get_rtp_seq_num (buf)); + + expected_ts = dts_base + i * TEST_BUF_DURATION; + fail_unless_equals_int (expected_ts, GST_BUFFER_PTS (buf)); + gst_buffer_unref (buf); + } + + /* generate a gap of 1 */ + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, + generate_test_buffer_full (dts_base + + num_consecutive * TEST_BUF_DURATION, + seqnum_base + num_consecutive + 1, + rtpts_base + num_consecutive * TEST_RTP_TS_DURATION))); + + /* demand we generated a lost event or RTX request with the correct pts */ + expected_seqnum = seqnum_base + num_consecutive; + expected_ts = dts_base + (num_consecutive * TEST_BUF_DURATION); + gst_harness_crank_single_clock_wait (h); + if (do_rtx) { + gst_harness_crank_single_clock_wait (h); + verify_rtx_event (h, expected_seqnum, expected_ts, 100, TEST_BUF_DURATION); + } else { + verify_lost_event (h, expected_seqnum, expected_ts, 0); + } + + gst_harness_teardown (h); +} + +GST_START_TEST (test_large_packet_spacing_lost_pkt_pts) +{ + test_large_packet_spacing_event_pkt_pts (FALSE); +} + +GST_END_TEST; + + +GST_START_TEST (test_large_packet_spacing_rtx) +{ + test_large_packet_spacing_event_pkt_pts (TRUE); +} + +GST_END_TEST; + typedef struct { guint seqnum_offset; @@ -2924,6 +3038,9 @@ GST_START_TEST (test_considered_lost_packet_in_large_gap_arrives) testclock = gst_harness_get_testclock (h); g_object_set (h->element, "do-lost", TRUE, "latency", jb_latency_ms, NULL); + /* pull out the latency-changed event */ + gst_event_unref (gst_harness_pull_event (h)); + /* first push buffer 0 */ fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, generate_test_buffer_full (0 * TEST_BUF_DURATION, @@ -2973,6 +3090,40 @@ GST_START_TEST (test_considered_lost_packet_in_large_gap_arrives) GST_END_TEST; +GST_START_TEST (test_latency_changed_event) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + GstEvent *event; + + fail_unless_equals_int (0, gst_harness_events_received (h)); + + /* changing the latency should cause a downstream latency-changed event */ + g_object_set (h->element, "latency", 20, NULL); + + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_LATENCY_CHANGED, GST_EVENT_TYPE (event)); + gst_event_unref (event); + + fail_unless_equals_int (1, gst_harness_events_received (h)); + + /* same latency does not send the event again */ + g_object_set (h->element, "latency", 20, NULL); + fail_unless_equals_int (1, gst_harness_events_received (h)); + + /* and changing again causes another one */ + g_object_set (h->element, "latency", 40, NULL); + + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_LATENCY_CHANGED, GST_EVENT_TYPE (event)); + gst_event_unref (event); + + fail_unless_equals_int (2, gst_harness_events_received (h)); + + gst_harness_teardown (h); +} + +GST_END_TEST; + GST_START_TEST (test_performance) { GstHarness *h = @@ -3044,39 +3195,34 @@ typedef struct { gint64 dts_skew; gint16 seqnum_skew; -} RtxSkewCtx; +} JitterBufferSkewCtx; -static const RtxSkewCtx rtx_does_not_affect_pts_calculation_input[] = { - {0, 0}, - {20 * GST_MSECOND, -100}, - {20 * GST_MSECOND, 100}, - {-10 * GST_MSECOND, 1}, - {100 * GST_MSECOND, 0}, -}; - -GST_START_TEST (test_rtx_does_not_affect_pts_calculation) +static void +test_redundant_pkt_does_not_affect_pts_calculation (GstRTPBufferFlags flags, + const JitterBufferSkewCtx * ctx) { GstHarness *h = gst_harness_new ("rtpjitterbuffer"); GstBuffer *buffer; guint next_seqnum; - guint rtx_seqnum; + guint redundant_pkt_seqnum; GstClockTime now; - const RtxSkewCtx *ctx = &rtx_does_not_affect_pts_calculation_input[__i__]; /* set up a deterministic state and take the time on the clock */ - g_object_set (h->element, "do-retransmission", TRUE, "do-lost", TRUE, NULL); + if (flags & GST_RTP_BUFFER_FLAG_RETRANSMISSION) + g_object_set (h->element, "do-retransmission", TRUE, "do-lost", TRUE, NULL); + next_seqnum = construct_deterministic_initial_state (h, 3000); now = gst_clock_get_time (GST_ELEMENT_CLOCK (h->element)); - /* push in a "bad" RTX buffer, arriving at various times / seqnums */ - rtx_seqnum = next_seqnum + ctx->seqnum_skew; - buffer = generate_test_buffer_full (now + ctx->dts_skew, rtx_seqnum, - rtx_seqnum * TEST_RTP_TS_DURATION); - GST_BUFFER_FLAG_SET (buffer, GST_RTP_BUFFER_FLAG_RETRANSMISSION); + /* push in a "bad" redundant buffer, arriving at various times / seqnums */ + redundant_pkt_seqnum = next_seqnum + ctx->seqnum_skew; + buffer = generate_test_buffer_full (now + ctx->dts_skew, redundant_pkt_seqnum, + redundant_pkt_seqnum * TEST_RTP_TS_DURATION); + GST_BUFFER_FLAG_SET (buffer, flags); gst_harness_push (h, buffer); /* now push in the next regular buffer at its ideal time, and verify the - rogue RTX-buffer did not mess things up */ + rogue RTX or ULPFEC buffer did not mess things up */ push_test_buffer (h, next_seqnum); now = gst_clock_get_time (GST_ELEMENT_CLOCK (h->element)); buffer = gst_harness_pull (h); @@ -3087,6 +3233,22 @@ GST_START_TEST (test_rtx_does_not_affect_pts_calculation) gst_harness_teardown (h); } +static const JitterBufferSkewCtx rtx_does_not_affect_pts_calculation_input[] = { + {0, 0}, + {20 * GST_MSECOND, -100}, + {20 * GST_MSECOND, 100}, + {-10 * GST_MSECOND, 1}, + {100 * GST_MSECOND, 0}, +}; + +GST_START_TEST (test_rtx_does_not_affect_pts_calculation) +{ + const JitterBufferSkewCtx *ctx = + &rtx_does_not_affect_pts_calculation_input[__i__]; + test_redundant_pkt_does_not_affect_pts_calculation + (GST_RTP_BUFFER_FLAG_RETRANSMISSION, ctx); +} + GST_END_TEST; GST_START_TEST (test_dont_drop_packet_based_on_skew) @@ -3628,6 +3790,393 @@ GST_START_TEST (test_early_rtcp_sr_allows_meta) GST_END_TEST; +GST_START_TEST (test_ulpfec_large_pkt_spacing) +{ + gint latency_ms = 20; + gint frame_dur_ms = 50; + gint i, seq; + GstBuffer *buffer; + GstClockTime now; + GstClockTime frame_dur = frame_dur_ms * GST_MSECOND; + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + + g_object_set (h->element, "do-lost", TRUE, "latency", latency_ms, NULL); + gst_harness_set_src_caps (h, generate_caps ()); + + /* Pushing 2 frames @frame_dur_ms ms apart from each other to initialize + * packet_spacing and avg jitter */ + for (seq = 0, now = 0; seq < 2; ++seq, now += frame_dur) { + gst_harness_set_time (h, now); + gst_harness_push (h, generate_test_buffer_full (now, seq, + AS_TEST_BUF_RTP_TIME (now))); + if (seq == 0) + gst_harness_crank_single_clock_wait (h); + buffer = gst_harness_pull (h); + fail_unless_equals_int64 (now, GST_BUFFER_PTS (buffer)); + gst_buffer_unref (buffer); + } + + /* drop GstEventStreamStart & GstEventCaps & GstEventSegment & Latency */ + for (i = 0; i < 4; i++) + gst_event_unref (gst_harness_pull_event (h)); + /* drop reconfigure event */ + gst_event_unref (gst_harness_pull_upstream_event (h)); + + /* Pushing packet #2 as ULPFEC */ + now = seq * frame_dur; + gst_harness_set_time (h, now); + buffer = generate_test_buffer_full (now, seq, AS_TEST_BUF_RTP_TIME (now)); + GST_BUFFER_FLAG_SET (buffer, GST_RTP_BUFFER_FLAG_ULPFEC); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buffer)); + buffer = gst_harness_pull (h); + fail_unless_equals_int64 (now, GST_BUFFER_PTS (buffer)); + gst_buffer_unref (buffer); + fail_unless_equals_int (0, gst_harness_buffers_in_queue (h)); + + + /* Packet #3 should have PTS not affected by clock skew logic */ + seq += 1; + now = seq * frame_dur; + gst_harness_set_time (h, now); + gst_harness_push (h, generate_test_buffer_full (now, seq, + AS_TEST_BUF_RTP_TIME (now))); + buffer = gst_harness_pull (h); + fail_unless_equals_int64 (now, GST_BUFFER_PTS (buffer)); + gst_buffer_unref (buffer); + + gst_harness_teardown (h); +} + +GST_END_TEST; + + +static const JitterBufferSkewCtx + ulpfec_pkt_does_not_affect_pts_calculation_input[] = { + {0, 0}, + {20 * GST_MSECOND, -100}, + {-10 * GST_MSECOND, 1}, + {100 * GST_MSECOND, 0}, +}; + +GST_START_TEST (test_ulpfec_does_not_affect_pts_calculation) +{ + const JitterBufferSkewCtx *ctx = + &ulpfec_pkt_does_not_affect_pts_calculation_input[__i__]; + + test_redundant_pkt_does_not_affect_pts_calculation + (GST_RTP_BUFFER_FLAG_ULPFEC, ctx); +} + +GST_END_TEST; + +typedef struct +{ + guint start_seqnum; + gint32 rtptime_diff_in_seqnum; +} JitterBufferSkewRtxCtx; + +static const JitterBufferSkewRtxCtx + ulpfec_jump_of_rtp_ts_does_not_drop_pkt_input[] = { + {0, 0}, + {100, 0}, + {100, 100}, + {100, 200}, + {100, -151}, + {100, -152}, + {100, -200} +}; + +GST_START_TEST (test_ulpfec_jump_of_rtp_ts_does_not_drop_pkt) +{ + const JitterBufferSkewRtxCtx *ctx = + &ulpfec_jump_of_rtp_ts_does_not_drop_pkt_input[__i__]; + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + GstBuffer *buffer = NULL; + guint next_seqnum = 0; + GstClockTime now = 0; + + /* Init jitterbuffer from N-th packet (value of 'next_seqnum'). That would make base_time of jitterbuffer + * starting from (N * TEST_BUF_MS) */ + next_seqnum = + construct_deterministic_initial_state_full (h, ctx->start_seqnum, 3000, + generate_caps ()); + now = gst_clock_get_time (GST_ELEMENT_CLOCK (h->element)); + + /* push in a "bad" ULPFEC buffer with skewed RTP timestamp */ + buffer = generate_test_buffer_full (now, next_seqnum, + ((next_seqnum + ctx->rtptime_diff_in_seqnum) * TEST_RTP_TS_DURATION)); + + GST_BUFFER_FLAG_SET (buffer, GST_RTP_BUFFER_FLAG_ULPFEC); + + gst_harness_push (h, buffer); + + /* expect ULPFEC buffer to pass through the rtpjitterbuffer */ + buffer = gst_harness_pull (h); + gst_buffer_unref (buffer); + next_seqnum++; + + /* now push in the next regular buffer at its ideal time, and verify the + rogue ULPFEC buffer did not mess things up */ + push_test_buffer (h, next_seqnum); + now = gst_clock_get_time (GST_ELEMENT_CLOCK (h->element)); + buffer = gst_harness_pull (h); + fail_unless_equals_int (next_seqnum, get_rtp_seq_num (buffer)); + fail_unless_equals_int64 (now, GST_BUFFER_PTS (buffer)); + + gst_buffer_unref (buffer); + + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_find_random_stall) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + guint i; + guint num_bufs = 10; + BufferArrayCtx *ctx = g_new0 (BufferArrayCtx, num_bufs); + for (i = 0; i < num_bufs; i++) { + ctx[i].seqnum_d = g_random_int_range (0, 300) - 20; + ctx[i].rtptime_d = g_random_int_range (0, 300) - 20; + ctx[i].rtptime_d *= TEST_RTP_TS_DURATION; + ctx[i].rtx = g_random_boolean (); + ctx[i].sleep_us = g_random_int_range (0, G_USEC_PER_SEC / 5); + } + + g_object_set (h->element, "do-lost", TRUE, + "do-retransmission", TRUE, + "rtx-next-seqnum", FALSE, + "rtx-delay-reorder", 0, + "rtx-max-retries", 5, + "rtx-min-retry-timeout", 50, "rtx-retry-period", 250, NULL); + + if (!check_for_stall (h, ctx, num_bufs)) { + for (i = 0; i < num_bufs; i++) { + g_print (" { %d, %d, %s, %d },\n", + ctx[i].seqnum_d, ctx[i].rtptime_d, + ctx[i].rtx ? "TRUE" : "FALSE", ctx[i].sleep_us); + } + fail_if (TRUE); + } + g_free (ctx); + + gst_harness_teardown (h); +} + +GST_END_TEST; + +static void +validate_estimated_latency (GstHarness * h, guint expected_estimated_latency_ms) +{ + GstStructure *s = NULL; + gint diff = 0; + guint actual_estimated_latency_ms = 0; + g_object_get (h->element, "stats", &s, NULL); + fail_unless (gst_structure_get_uint (s, "estimated-latency-ms", + &actual_estimated_latency_ms)); + + diff = (actual_estimated_latency_ms - expected_estimated_latency_ms); + fail_unless ((diff == 0), + "Actual 'estimated-latency-ms' (%" GST_TIME_FORMAT + ") is off from expected value (%" GST_TIME_FORMAT ") " "by %" + GST_STIME_FORMAT ".", + GST_TIME_ARGS (actual_estimated_latency_ms * GST_MSECOND), + GST_TIME_ARGS (expected_estimated_latency_ms * GST_MSECOND), + GST_STIME_ARGS (diff * GST_MSECOND)); + gst_structure_free (s); +} + + +static gint +buffer_array_append_sequential_with_jitter (GArray * array, + gint * jitter_array_ms, guint num_bufs, gint prev_jitter) +{ + guint i; + for (i = 0; i < num_bufs; i++) { + BufferArrayCtx ctx; + ctx.seqnum_d = 1; + ctx.rtptime_d = TEST_RTP_TS_DURATION; /* 20ms for 8KHz */ + ctx.rtx = FALSE; + ctx.sleep_us = + (G_USEC_PER_SEC / 1000 * (TEST_BUF_MS - prev_jitter + + jitter_array_ms[i])); + g_array_append_val (array, ctx); + prev_jitter = jitter_array_ms[i]; + } + return prev_jitter; +} + +static void +buffer_array_push_fast (GstHarness * h, GArray * array, + guint16 seqnum_base, guint32 rtptime_base) +{ + guint16 seqnum = seqnum_base; + guint32 rtptime = rtptime_base; + GstClockTime now = gst_clock_get_time (GST_ELEMENT_CLOCK (h->element)); + guint i; + + for (i = 0; i < array->len; i++) { + BufferArrayCtx *ctx = &g_array_index (array, BufferArrayCtx, i); + GstBuffer *buf = generate_test_buffer_full (now, seqnum, rtptime); + if (ctx->rtx) + GST_BUFFER_FLAG_SET (buf, GST_RTP_BUFFER_FLAG_RETRANSMISSION); + fail_unless_equals_int (GST_FLOW_OK, gst_harness_push (h, buf)); + seqnum += ctx->seqnum_d; + rtptime += ctx->rtptime_d; + now += (GST_USECOND * ctx->sleep_us); + } +} + +static void +push_buf_array (GstHarness * h, GArray * array) +{ + guint16 base_seqnum = 10000; + guint32 base_rtptime = base_seqnum * TEST_RTP_TS_DURATION; + gst_harness_set_src_caps (h, generate_caps ()); + buffer_array_push_fast (h, array, base_seqnum, base_rtptime); +} + +GST_START_TEST (test_latency_estimation_no_jitter) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + GArray *array = g_array_new (FALSE, FALSE, sizeof (BufferArrayCtx)); + + g_object_set (h->element, "latency", 0, NULL); + buffer_array_append_sequential (array, 128); + push_buf_array (h, array); + validate_estimated_latency (h, 0); + + g_array_set_size (array, 0); + g_array_unref (array); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_latency_estimation_with_jitter) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + gint jitter_array_ms[] = { 1, 4, -3, 5, 3, -2, 7 }; + GArray *array = g_array_new (FALSE, FALSE, sizeof (BufferArrayCtx)); + + g_object_set (h->element, "latency", 0, NULL); + + buffer_array_append_sequential (array, 128); + /* Push packets with jitter and validate latency estimation */ + buffer_array_append_sequential_with_jitter (array, jitter_array_ms, + G_N_ELEMENTS (jitter_array_ms), 0); + push_buf_array (h, array); + validate_estimated_latency (h, 7); + g_array_set_size (array, 0); + + /* Push more packets without jitter and validate + * latency estimation decreasing */ + buffer_array_append_sequential (array, 256); + push_buf_array (h, array); + validate_estimated_latency (h, 5); + + g_array_set_size (array, 0); + g_array_unref (array); + gst_harness_teardown (h); +} + +GST_END_TEST; + + +GST_START_TEST (test_latency_estimation_increasing_delay) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + gint jitter_array_ms[] = { 1 }; + GArray *array = g_array_new (FALSE, FALSE, sizeof (BufferArrayCtx)); + + g_object_set (h->element, "latency", 0, NULL); + + buffer_array_append_sequential (array, 128); + + for (size_t i = 0; i < 2000; i++) { + buffer_array_append_sequential_with_jitter (array, jitter_array_ms, + G_N_ELEMENTS (jitter_array_ms), 0); + } + + /* All buffers with ever-increasing delay */ + push_buf_array (h, array); + /* This is unusual usecase but latency estimation + * should not blow up */ + validate_estimated_latency (h, 127); + + g_array_set_size (array, 0); + g_array_unref (array); + gst_harness_teardown (h); +} + +GST_END_TEST; + + +GST_START_TEST (test_latency_estimation_reaction_on_outlyer) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + gint jitter_array_ms[] = { 1, -1, 100, -1, }; + GArray *array = g_array_new (FALSE, FALSE, sizeof (BufferArrayCtx)); + + g_object_set (h->element, "latency", 0, NULL); + + buffer_array_append_sequential (array, 128); + + buffer_array_append_sequential_with_jitter (array, jitter_array_ms, + G_N_ELEMENTS (jitter_array_ms), 0); + /* Push buffers with one outlayer (100ms late) */ + push_buf_array (h, array); + /* Validat latency estimation */ + validate_estimated_latency (h, 100); + + g_array_set_size (array, 0); + /* Push more buffers and check that we reduce + * latency estimation back to 0 */ + buffer_array_append_sequential (array, 1500); + push_buf_array (h, array); + validate_estimated_latency (h, 0); + + g_array_set_size (array, 0); + g_array_unref (array); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtcp_non_utf8_cname) +{ + GstHarness *h = gst_harness_new ("rtpjitterbuffer"); + GstHarness *h_rtcp = gst_harness_new_with_element (h->element, "sink_rtcp", NULL); + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + GstRTCPPacket packet; + GstBuffer *rtcp_buf; + guint8 cname[] = { 0xf0, 0x28, 0x8c, 0x28, 0xf0, 0x28, 0x8c, 0x28, 0x00 }; + + gst_harness_set_src_caps (h, generate_caps ()); + gst_harness_set_src_caps_str (h_rtcp, "application/x-rtcp"); + + /* push in a SDES with an non-utf8 cname */ + rtcp_buf = gst_rtcp_buffer_new (1000); + fail_unless (gst_rtcp_buffer_map (rtcp_buf, GST_MAP_READWRITE, &rtcp)); + fail_unless (gst_rtcp_buffer_add_packet (&rtcp, GST_RTCP_TYPE_SR, &packet)); + gst_rtcp_packet_sr_set_sender_info (&packet, TEST_BUF_SSRC, 0, 0, 0, 0); + fail_unless (gst_rtcp_buffer_add_packet (&rtcp, GST_RTCP_TYPE_SDES, + &packet)); + fail_unless (gst_rtcp_packet_sdes_add_item (&packet, TEST_BUF_SSRC)); + fail_unless (gst_rtcp_packet_sdes_add_entry (&packet, GST_RTCP_SDES_CNAME, + sizeof (cname), cname)); + gst_rtcp_buffer_unmap (&rtcp); + gst_harness_push (h_rtcp, rtcp_buf); + + /* now push in a normal buffer, and don't explode */ + gst_harness_push (h, generate_test_buffer (0)); + + gst_harness_teardown (h_rtcp); + gst_harness_teardown (h); +} +GST_END_TEST; + static Suite * rtpjitterbuffer_suite (void) { @@ -3688,14 +4237,19 @@ rtpjitterbuffer_suite (void) tcase_add_test (tc_chain, test_deadline_ts_offset); tcase_add_test (tc_chain, test_deadline_ts_offset_overflow); - tcase_add_test (tc_chain, test_big_gap_seqnum); + tcase_add_loop_test (tc_chain, test_big_gap_seqnum, 0, + G_N_ELEMENTS (big_gap_testdata)); tcase_add_test (tc_chain, test_big_gap_arrival_time); + tcase_add_test (tc_chain, test_large_packet_spacing_lost_pkt_pts); + tcase_add_test (tc_chain, test_large_packet_spacing_rtx); tcase_add_test (tc_chain, test_fill_queue); tcase_add_loop_test (tc_chain, test_considered_lost_packet_in_large_gap_arrives, 0, G_N_ELEMENTS (test_considered_lost_packet_in_large_gap_arrives_input)); + tcase_add_test (tc_chain, test_latency_changed_event); + tcase_add_test (tc_chain, test_performance); tcase_add_test (tc_chain, test_drop_messages_too_late); @@ -3710,6 +4264,20 @@ rtpjitterbuffer_suite (void) tcase_add_test (tc_chain, test_early_rtcp_sr_allows_meta); + tcase_add_test (tc_chain, test_ulpfec_large_pkt_spacing); + tcase_add_loop_test (tc_chain, test_ulpfec_jump_of_rtp_ts_does_not_drop_pkt, + 0, G_N_ELEMENTS (ulpfec_jump_of_rtp_ts_does_not_drop_pkt_input)); + tcase_add_loop_test (tc_chain, test_ulpfec_does_not_affect_pts_calculation, 0, + G_N_ELEMENTS (ulpfec_pkt_does_not_affect_pts_calculation_input)); + + tcase_add_test (tc_chain, test_find_random_stall); + tcase_add_test (tc_chain, test_latency_estimation_no_jitter); + tcase_add_test (tc_chain, test_latency_estimation_with_jitter); + tcase_add_test (tc_chain, test_latency_estimation_increasing_delay); + tcase_add_test (tc_chain, test_latency_estimation_reaction_on_outlyer); + + tcase_add_test (tc_chain, test_rtcp_non_utf8_cname); + return s; } diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpmux.c b/subprojects/gst-plugins-good/tests/check/elements/rtpmux.c index c70018f22eb..aa61c589d5e 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpmux.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpmux.c @@ -40,7 +40,7 @@ static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src", typedef void (*check_cb) (GstPad * pad, int i); static gboolean -query_func (GstPad * pad, GstObject * noparent, GstQuery * query) +query_func (GstPad * pad, G_GNUC_UNUSED GstObject * noparent, GstQuery * query) { switch (GST_QUERY_TYPE (query)) { case GST_QUERY_CAPS: @@ -71,7 +71,7 @@ remove_ssrc_from_caps (GstCaps * caps) } static gboolean -event_func (GstPad * pad, GstObject * noparent, GstEvent * event) +event_func (GstPad * pad, G_GNUC_UNUSED GstObject * noparent, GstEvent * event) { switch (GST_EVENT_TYPE (event)) { case GST_EVENT_CAPS: @@ -208,8 +208,7 @@ test_basic (const gchar * elem_name, const gchar * sink2, int count, fail_unless (gst_pad_push (src1, inbuf) == GST_FLOW_OK); if (buffers) - fail_unless (GST_BUFFER_PTS (buffers->data) == i * 1000, "%lld", - GST_BUFFER_PTS (buffers->data)); + fail_unless_equals_uint64 (GST_BUFFER_PTS (buffers->data), i * 1000); cb (src2, i); @@ -241,7 +240,7 @@ test_basic (const gchar * elem_name, const gchar * sink2, int count, } static void -basic_check_cb (GstPad * pad, int i) +basic_check_cb (G_GNUC_UNUSED GstPad * pad, int i) { GstRTPBuffer rtpbuffer = GST_RTP_BUFFER_INIT; fail_unless (buffers && g_list_length (buffers) == 1); diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c b/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c index 1580c8bf4a9..890affb1773 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtprtx.c @@ -131,9 +131,9 @@ create_rtp_buffer_ex (guint32 ssrc, guint8 payload_type, guint16 seqnum, } static GstBuffer * -create_rtp_buffer (guint32 ssrc, guint8 payload_type, guint16 seqnum) +create_rtp_buffer_with_payload_size (guint32 ssrc, guint8 payload_type, + guint16 seqnum, guint payload_size) { - guint payload_size = 29; guint64 timestamp = gst_util_uint64_scale_int (seqnum, 90000, 30); GstRTPBuffer *rtpbuf = create_rtp_buffer_ex (ssrc, payload_type, seqnum, (guint32) timestamp, payload_size); @@ -146,6 +146,61 @@ create_rtp_buffer (guint32 ssrc, guint8 payload_type, guint16 seqnum) return ret; } +static GstBuffer * +create_rtp_buffer_with_twcc_ext (guint32 ssrc, guint8 payload_type, + guint16 seqnum, guint payload_size, guint8 twcc_ext_id, guint16 twcc_seqnum) +{ + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + GstBuffer *buf = + create_rtp_buffer_with_payload_size (ssrc, payload_type, seqnum, + payload_size); + + gst_rtp_buffer_map (buf, GST_MAP_READWRITE, &rtp); + + if (twcc_ext_id > 0) { + guint8 twcc_seqnum_be[2]; + GST_WRITE_UINT16_BE (twcc_seqnum_be, twcc_seqnum); + gst_rtp_buffer_add_extension_onebyte_header (&rtp, twcc_ext_id, + twcc_seqnum_be, sizeof (twcc_seqnum_be)); + } + + gst_rtp_buffer_unmap (&rtp); + + return buf; +} + +static gint16 +read_twcc_seqnum (GstBuffer * buf, guint8 twcc_ext_id) +{ + gint16 twcc_seqnum; + gpointer ext_data; + guint ext_size; + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + + if (!gst_rtp_buffer_map (buf, GST_MAP_READ, &rtp)) + return -1; + + if (!gst_rtp_buffer_get_extension_onebyte_header (&rtp, twcc_ext_id, + 0, &ext_data, &ext_size)) { + gst_rtp_buffer_unmap (&rtp); + return -1; + } + + fail_unless (ext_data != NULL); + fail_unless (ext_size == 2); + + twcc_seqnum = GST_READ_UINT16_BE (ext_data); + gst_rtp_buffer_unmap (&rtp); + + return twcc_seqnum; +} + +static GstBuffer * +create_rtp_buffer (guint32 ssrc, guint8 payload_type, guint16 seqnum) +{ + return create_rtp_buffer_with_payload_size (ssrc, payload_type, seqnum, 29); +} + static GstBuffer * create_rtp_buffer_with_timestamp (guint32 ssrc, guint8 payload_type, guint16 seqnum, guint32 timestamp, GstClockTime pts) @@ -1035,6 +1090,191 @@ GST_START_TEST (test_rtxsend_header_extensions_copy) GST_END_TEST; +GST_START_TEST (test_rtxsender_copy_twcc_exthdr) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstBuffer *rtx_buf; + guint8 twcc_ext_id = 5; + GstHarness *h = gst_harness_new ("rtprtxsend"); + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 800, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + gst_harness_set_time (h, 0 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_twcc_ext (master_ssrc, master_pt, 0, 20, + twcc_ext_id, 33)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + + gst_harness_push_upstream_event (h, create_rtx_event (master_ssrc, + master_pt, 0)); + + rtx_buf = gst_harness_pull (h); + verify_buf (rtx_buf, TRUE, rtx_ssrc, rtx_pt, 0); + fail_unless (read_twcc_seqnum (rtx_buf, twcc_ext_id) == 33); + gst_buffer_unref (rtx_buf); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + +static GstBuffer * +create_rtp_buffer_with_roi_ext (guint32 ssrc, guint8 payload_type, + guint16 seqnum, guint payload_size, guint8 ext_id, + guint16 x, guint16 y, guint16 w, guint16 h, guint8 id, guint16 num_faces) +{ + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + GstBuffer *buf = + create_rtp_buffer_with_payload_size (ssrc, payload_type, seqnum, + payload_size); + + gst_rtp_buffer_map (buf, GST_MAP_READWRITE, &rtp); + + if (ext_id > 0) { + guint8 data[11]; + GST_WRITE_UINT16_BE (&data[0], x); + GST_WRITE_UINT16_BE (&data[2], y); + GST_WRITE_UINT16_BE (&data[4], w); + GST_WRITE_UINT16_BE (&data[6], h); + GST_WRITE_UINT16_BE (&data[9], num_faces); + GST_WRITE_UINT8 (&data[8], id); + gst_rtp_buffer_add_extension_twobytes_header (&rtp, 0, ext_id, + data, sizeof (data)); + } + + gst_rtp_buffer_unmap (&rtp); + + return buf; +} + +static void +read_roi (GstBuffer * buf, guint8 ext_id, + guint16 * x, guint16 * y, guint16 * w, guint16 * h, + guint8 * id, guint16 * num_faces) +{ + gpointer ext_data; + guint ext_size; + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + + if (!gst_rtp_buffer_map (buf, GST_MAP_READ, &rtp)) + return; + + if (!gst_rtp_buffer_get_extension_twobytes_header (&rtp, NULL, ext_id, 0, + &ext_data, &ext_size)) { + gst_rtp_buffer_unmap (&rtp); + return; + } + + fail_unless (ext_data != NULL); + fail_unless (ext_size == 11); + + { + const guint8 *bytes = (const guint8 *) ext_data; + *x = GST_READ_UINT16_BE (&bytes[0]); + *y = GST_READ_UINT16_BE (&bytes[2]); + *w = GST_READ_UINT16_BE (&bytes[4]); + *h = GST_READ_UINT16_BE (&bytes[6]); + *id = GST_READ_UINT8 (&bytes[8]); + *num_faces = GST_READ_UINT16_BE (&bytes[9]); + } + gst_rtp_buffer_unmap (&rtp); +} + +GST_START_TEST (test_rtxsender_copy_roi_exthdr) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstBuffer *inbuf, *outbuf, *rtxbuf; + guint8 ext_id = 5; + GstHarness *hsend = gst_harness_new ("rtprtxsend"); + GstHarness *hrecv = gst_harness_new ("rtprtxreceive"); + guint16 expected_x = 2, actual_x = 0; + guint16 expected_y = 3, actual_y = 0; + guint16 expected_w = 5, actual_w = 0; + guint16 expected_h = 7, actual_h = 0; + guint8 expected_id = 11, actual_id = 0; + guint16 expected_num_faces = 13, actual_num_faces = 0; + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (hsend->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 800, NULL); + g_object_set (hrecv->element, + "payload-type-map", pt_map, "ssrc-map", ssrc_map, NULL); + + gst_harness_set_src_caps_str (hsend, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + gst_harness_set_src_caps_str (hrecv, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + gst_harness_set_time (hsend, 0 * GST_MSECOND); + gst_harness_set_time (hrecv, 0 * GST_MSECOND); + + /* create buffer with RoI extension on it */ + inbuf = create_rtp_buffer_with_roi_ext (master_ssrc, master_pt, 0, 20, + ext_id, expected_x, expected_y, expected_w, expected_h, + expected_id, expected_num_faces); + + /* push packets through rtxsend to rtxreceive */ + gst_harness_push (hsend, gst_buffer_ref (inbuf)); + gst_harness_push (hrecv, gst_harness_pull (hsend)); + pull_and_verify (hrecv, FALSE, master_ssrc, master_pt, 0); + + /* get rid of reconfigure event in preparation for next step */ + gst_event_unref (gst_harness_pull_upstream_event (hrecv)); + fail_unless_equals_int (gst_harness_upstream_events_in_queue (hrecv), 0); + + /* push RTX events through rtxsend to rtxreceive */ + gst_harness_push_upstream_event (hrecv, + create_rtx_event (master_ssrc, master_pt, 0)); + gst_harness_push_upstream_event (hsend, + gst_harness_pull_upstream_event (hrecv)); + + rtxbuf = gst_harness_pull (hsend); + verify_buf (rtxbuf, TRUE, rtx_ssrc, rtx_pt, 0); + read_roi (rtxbuf, ext_id, &actual_x, &actual_y, &actual_w, &actual_h, + &actual_id, &actual_num_faces); + fail_unless_equals_int (expected_x, actual_x); + fail_unless_equals_int (expected_y, actual_y); + fail_unless_equals_int (expected_w, actual_w); + fail_unless_equals_int (expected_h, actual_h); + fail_unless_equals_int (expected_id, actual_id); + fail_unless_equals_int (expected_num_faces, actual_num_faces); + gst_harness_push (hrecv, rtxbuf); + + outbuf = gst_harness_pull (hrecv); + compare_rtp_packets (inbuf, outbuf); + gst_buffer_unref (inbuf); + gst_buffer_unref (outbuf); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (hsend); + gst_harness_teardown (hrecv); +} + +GST_END_TEST; #define RTPHDREXT_STREAM_ID GST_RTP_HDREXT_BASE "sdes:rtp-stream-id" #define RTPHDREXT_REPAIRED_STREAM_ID GST_RTP_HDREXT_BASE "sdes:repaired-rtp-stream-id" @@ -1052,7 +1292,7 @@ GST_START_TEST (test_rtxsend_header_extensions) GstRTPHeaderExtension *send_stream_id, *send_repaired_stream_id; GstRTPHeaderExtension *recv_stream_id, *recv_repaired_stream_id; guint stream_hdr_id = 1, repaired_hdr_id = 2; - gint i; + guint i; pt_map = gst_structure_new ("application/x-rtp-pt-map", "96", G_TYPE_UINT, rtx_pt, NULL); @@ -1162,6 +1402,482 @@ GST_START_TEST (test_rtxsend_header_extensions) GST_END_TEST; +GST_START_TEST (test_rtxsender_stuffing) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 800, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + /* 800 kbps = 100 kBps. 10 ms distance between each packet means 100 packets + * per second and 1000 bytes per packet to hit target kbps. + * We stuff considering the payload length. */ + gst_harness_set_time (h, 0 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + + gst_harness_set_time (h, 10 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 1, 580)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 1); + /* budget 1000, sent 600, no stuffing (stuff with 580 will exceed budget) */ + + gst_harness_set_time (h, 20 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 600)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + + gst_harness_set_time (h, 30 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 3, 200)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 3); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 3); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 3); + + gst_harness_set_time (h, 40 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 4, 600)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 4); + g_usleep (G_USEC_PER_SEC / 100); + fail_if (gst_harness_try_pull (h)); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + +/* +* This tests verify that the element doesn't generate stuffing buffers, unless +* the property is set to non zero. +*/ +GST_START_TEST (test_rtxsender_stuffing_toggle) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 0, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + gst_harness_set_time (h, 0 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + + gst_harness_set_time (h, 10 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 1, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 1); + + gst_harness_set_time (h, 20 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); + + /* 80 kbps = 10 kBps. 10 ms distance between each packet means 10 packets + * per second and 100 bytes per packet to hit target kbps. + * We stuff considering the payload length. */ + g_object_set (h->element, "stuffing-kbps", 245, NULL); + + gst_harness_set_time (h, 30 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 3, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 3); + + gst_harness_set_time (h, 40 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 4, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 4); + + /* budget 100, sent 40, so stuff with #2, 3 and 4, total of 60 bytes */ + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 3); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 4); + + /* disable stuffing */ + g_object_set (h->element, "stuffing-kbps", 0, NULL); + + gst_harness_set_time (h, 50 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 5, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 5); + + g_usleep (G_USEC_PER_SEC / 100); + fail_if (gst_harness_try_pull (h)); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + +/* +* This tests verify that buffers with payload types that are not marked for +* RTX, also generate stuffing. +*/ +GST_START_TEST (test_rtxsender_stuffing_non_rtx_packets) +{ + guint video_ssrc = 1234567; + guint video_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + guint audio_ssrc = 6789123; + guint audio_pt = 111; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 800, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + /* 800 kbps = 100 kBps. 10 ms distance between each packet means 100 packets + * per second and 1000 bytes per packet to hit target kbps. + * We stuff considering the payload length. */ + gst_harness_set_time (h, 0 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (video_ssrc, video_pt, 0, 20)); + pull_and_verify (h, FALSE, video_ssrc, video_pt, 0); + + gst_harness_set_time (h, 10 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (video_ssrc, video_pt, 1, 580)); + pull_and_verify (h, FALSE, video_ssrc, video_pt, 1); + /* budget 1000, sent 600, no stuffing (stuff with 580 will exceed budget) */ + + gst_harness_set_time (h, 20 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (video_ssrc, video_pt, 2, 600)); + pull_and_verify (h, FALSE, video_ssrc, video_pt, 2); + /* budget 2000, sent 1200, stuff with #2, 600 bytes */ + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + + gst_harness_set_time (h, 30 * GST_MSECOND); + /* audio packet comes in.. */ + gst_harness_push (h, + create_rtp_buffer_with_payload_size (audio_ssrc, audio_pt, 0, 200)); + pull_and_verify (h, FALSE, audio_ssrc, audio_pt, 0); + /* budget 3000, sent 2000, stuff with #2, 600 bytes */ + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + + gst_harness_push (h, + create_rtp_buffer_with_payload_size (video_ssrc, video_pt, 4, 600)); + pull_and_verify (h, FALSE, video_ssrc, video_pt, 4); + /* budget 4000, sent 2600, no more stuffing */ + g_usleep (G_USEC_PER_SEC / 100); + fail_if (gst_harness_try_pull (h)); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + +/* +* This tests verify that buffers over the window size time are not included for stuffing. +*/ +GST_START_TEST (test_rtxsender_stuffing_window) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + GstBuffer *buf; + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 80, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + /* 80 kbps = 10 kBps. 10 ms distance between each packet means 10 packets + * per second and 100 bytes per packet to hit target kbps. + * We stuff considering the payload length. */ + gst_harness_set_time (h, 0 * GST_MSECOND); + buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 80); + GST_BUFFER_PTS (buf) = 0; + gst_harness_push (h, buf); + + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + /* budget 100, sent 80 */ + + gst_harness_set_time (h, 10 * GST_MSECOND); + buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 1, 80); + GST_BUFFER_PTS (buf) = 10 * GST_MSECOND; + gst_harness_push (h, buf); + + pull_and_verify (h, FALSE, master_ssrc, master_pt, 1); + /* budget 200, sent 160, no stuffing */ + + gst_harness_set_time (h, 110 * GST_MSECOND); + buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 80); + GST_BUFFER_PTS (buf) = 110 * GST_MSECOND; + gst_harness_push (h, buf); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); + + /* we have budget, so stuff with #1 and #2, but #0 is too old */ + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 1); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + + g_usleep (G_USEC_PER_SEC / 100); + fail_if (gst_harness_try_pull (h)); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + +/* +* This tests verify that even though there is no packets under the stuffing +* window, the element still produces stuffing. +*/ +GST_START_TEST (test_rtxsender_stuffing_no_packets_under_window) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + GstBuffer *buf; + gint i; + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 80, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + /* 80 kbps = 10 kBps. 10 ms distance between each packet means 10 packets + * per second and 100 bytes per packet to hit target kbps. + * We stuff considering the payload length. */ + gst_harness_set_time (h, 0 * GST_MSECOND); + buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 80); + GST_BUFFER_PTS (buf) = 0; + gst_harness_push (h, buf); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + /* budget 100, sent 80 */ + + gst_harness_set_time (h, 10 * GST_MSECOND); + buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 1, 80); + GST_BUFFER_PTS (buf) = 10 * GST_MSECOND; + gst_harness_push (h, buf); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 1); + /* budget 200, sent 160, no stuffing */ + + /* advance far ahead, so the buffers are out of the window */ + gst_harness_set_time (h, 150 * GST_MSECOND); + buf = create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 80); + GST_BUFFER_PTS (buf) = 20 * GST_MSECOND; + gst_harness_push (h, buf); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); + + /* we have budget, non of the above are under the window, so we stuff + repeatedly with the latest buffer */ + for (i = 0; i < 4; i++) { + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 2); + } + + g_usleep (G_USEC_PER_SEC / 100); + fail_if (gst_harness_try_pull (h)); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtxsender_stuffing_sanity_when_input_rate_is_extreme) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + gint i; + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 1, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + /* Produce an insane amount of data so that variables may overflow (more + * than G_MAXINT bits). No stuffing should be produced. */ + for (i = 0; i < 300; i++) { + gst_harness_set_time (h, i * GST_NSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, i, + 1000000)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, i); + } + fail_if (gst_harness_try_pull (h)); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtxsender_stuffing_does_not_interfer_with_rtx) +{ + /* Stuffing packets should be considered a part of the original data in + * terms of counting bits and should not interfer with rtx-kbps */ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, "stuffing-kbps", 800, "max-kbps", 400, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + gst_harness_set_time (h, 0 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 100)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + + gst_harness_set_time (h, 10 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 1, 500)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 1); + /* budget 1000, sent 600 */ + + /* Request and send RTX packet. Not calculated in budget */ + gst_harness_push_upstream_event (h, + create_rtx_event (master_ssrc, master_pt, 1)); + pull_and_verify (h, TRUE, rtx_ssrc, rtx_pt, 1); + + /* Send media packet. Budget should still be untouched for before. */ + gst_harness_set_time (h, 20 * GST_MSECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 2, 400)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 2); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + +static guint MAX_BURST_PACKETS[] = { + 5, 10, 100, 500, 777, +}; + +GST_START_TEST (test_rtxsender_stuffing_max_burst_packets) +{ + guint master_ssrc = 1234567; + guint master_pt = 96; + guint rtx_ssrc = 7777777; + guint rtx_pt = 99; + GstStructure *pt_map, *ssrc_map; + GstHarness *h = gst_harness_new ("rtprtxsend"); + guint max_burst_packets = MAX_BURST_PACKETS[__i__]; + + pt_map = gst_structure_new ("application/x-rtp-pt-map", + "96", G_TYPE_UINT, rtx_pt, NULL); + ssrc_map = gst_structure_new ("application/x-rtp-ssrc-map", + "1234567", G_TYPE_UINT, rtx_ssrc, NULL); + g_object_set (h->element, + "payload-type-map", pt_map, + "ssrc-map", ssrc_map, + "stuffing-kbps", 2000, + "stuffing-max-burst-packets", max_burst_packets, NULL); + + gst_harness_set_src_caps_str (h, "application/x-rtp, " + "payload = (int)96, " "ssrc = (uint)1234567, " "clock-rate = (int)90000"); + + gst_harness_set_time (h, 0 * GST_MSECOND); + /* push only a very small buffer to force the creation of a lot of stuffing */ + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 20)); + pull_and_verify (h, FALSE, master_ssrc, master_pt, 0); + + /* set the time 1 second in the future, to create a huge deficit on stuffing */ + gst_harness_set_time (h, 1 * GST_SECOND); + gst_harness_push (h, + create_rtp_buffer_with_payload_size (master_ssrc, master_pt, 0, 20)); + + /* verify we don't burst out more than the configured max-burst, plus the + one packet we pushed */ + fail_unless_equals_int (gst_harness_buffers_in_queue (h), + max_burst_packets + 1); + + gst_structure_free (pt_map); + gst_structure_free (ssrc_map); + gst_harness_teardown (h); +} + +GST_END_TEST; + static Suite * rtprtx_suite (void) { @@ -1189,6 +1905,21 @@ rtprtx_suite (void) tcase_add_test (tc_chain, test_rtxsender_clock_rate_map); tcase_add_test (tc_chain, test_rtxsend_header_extensions); tcase_add_test (tc_chain, test_rtxsend_header_extensions_copy); + tcase_add_test (tc_chain, test_rtxsender_copy_twcc_exthdr); + tcase_add_test (tc_chain, test_rtxsender_copy_roi_exthdr); + + tcase_add_test (tc_chain, test_rtxsender_stuffing); + tcase_add_test (tc_chain, test_rtxsender_stuffing_toggle); + tcase_add_test (tc_chain, test_rtxsender_stuffing_non_rtx_packets); + tcase_add_test (tc_chain, test_rtxsender_stuffing_window); + tcase_add_test (tc_chain, test_rtxsender_stuffing_no_packets_under_window); + tcase_add_test (tc_chain, + test_rtxsender_stuffing_sanity_when_input_rate_is_extreme); + tcase_add_test (tc_chain, test_rtxsender_stuffing_does_not_interfer_with_rtx); + + tcase_add_loop_test (tc_chain, test_rtxsender_stuffing_max_burst_packets, + 0, G_N_ELEMENTS (MAX_BURST_PACKETS)); + return s; } diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c index 60316fc9e18..9934b3bbef4 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtpsession.c @@ -4,6 +4,7 @@ * * Copyright (C) <2009> Wim Taymans * Copyright (C) 2013 Collabora Ltd. + * Copyright (C) <2018> Havard Graff * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -20,7 +21,9 @@ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ -#define GLIB_DISABLE_DEPRECATION_WARNINGS +#ifndef GLIB_DISABLE_DEPRECATION_WARNINGS +# define GLIB_DISABLE_DEPRECATION_WARNINGS +#endif #include #include @@ -29,9 +32,20 @@ #include #include +#include #include #include +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef HAVE_VALGRIND +# include +#else +# define RUNNING_ON_VALGRIND 0 +#endif + #define TEST_BUF_CLOCK_RATE 8000 #define TEST_BUF_PT 0 #define TEST_BUF_SSRC 0x01BADBAD @@ -44,12 +58,15 @@ #define TEST_TWCC_EXT_ID 5 #define TWCC_EXTMAP_STR "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01" +#define TEST_RTX_BUF_PT 100 +#define TEST_RTX_BUF_SSRC 0xd3add3ad + static GstCaps * -generate_caps (void) +generate_caps (guint8 payload_type) { return gst_caps_new_simple ("application/x-rtp", "clock-rate", G_TYPE_INT, TEST_BUF_CLOCK_RATE, - "payload", G_TYPE_INT, TEST_BUF_PT, NULL); + "payload", G_TYPE_INT, payload_type, NULL); } static GstBuffer * @@ -112,7 +129,7 @@ generate_twcc_send_buffer_full (guint seqnum, gboolean marker_bit, { return generate_test_buffer_full (seqnum * TEST_BUF_DURATION, seqnum, seqnum * TEST_RTP_TS_DURATION, ssrc, marker_bit, - payload_type, TEST_TWCC_EXT_ID, seqnum); + payload_type, 0, 0); } static GstBuffer * @@ -122,6 +139,92 @@ generate_twcc_send_buffer (guint seqnum, gboolean marker_bit) TEST_BUF_PT); } +static GstBuffer * +generate_rtx_buffer (guint rtx_seqnum, GstBuffer * buffer) +{ + GstMemory *mem = NULL; + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + GstRTPBuffer new_rtp = GST_RTP_BUFFER_INIT; + GstBuffer *new_buffer = gst_buffer_new (); + GstMapInfo map; + + gst_rtp_buffer_map (buffer, GST_MAP_READ, &rtp); + gst_rtp_buffer_get_payload (&rtp); + + /* copy fixed header */ + mem = gst_memory_copy (rtp.map[0].memory, 0, rtp.size[0]); + gst_buffer_append_memory (new_buffer, mem); + + /* copy extension if any */ + if (rtp.size[1]) { + mem = gst_allocator_alloc (NULL, rtp.size[1], NULL); + gst_memory_map (mem, &map, GST_MAP_WRITE); + memcpy (map.data, rtp.data[1], rtp.size[1]); + gst_memory_unmap (mem, &map); + gst_buffer_append_memory (new_buffer, mem); + } + + /* copy payload and add OSN just before */ + mem = gst_allocator_alloc (NULL, 2 + rtp.size[2], NULL); + + gst_memory_map (mem, &map, GST_MAP_WRITE); + GST_WRITE_UINT16_BE (map.data, gst_rtp_buffer_get_seq (&rtp)); + if (rtp.size[2]) + memcpy (map.data + 2, rtp.data[2], rtp.size[2]); + gst_memory_unmap (mem, &map); + gst_buffer_append_memory (new_buffer, mem); + + /* everything needed is copied */ + gst_rtp_buffer_unmap (&rtp); + + gst_rtp_buffer_map (new_buffer, GST_MAP_WRITE, &new_rtp); + gst_rtp_buffer_set_payload_type (&new_rtp, TEST_RTX_BUF_PT); + gst_rtp_buffer_set_ssrc (&new_rtp, TEST_RTX_BUF_SSRC); + gst_rtp_buffer_set_seq (&new_rtp, rtx_seqnum); + gst_rtp_buffer_unmap (&new_rtp); + + /* Copy over timestamps */ + gst_buffer_copy_into (new_buffer, buffer, GST_BUFFER_COPY_TIMESTAMPS, 0, -1); + + /* mark this is a RETRANSMISSION buffer */ + GST_BUFFER_FLAG_SET (new_buffer, GST_RTP_BUFFER_FLAG_RETRANSMISSION); + + return new_buffer; +} + +static gint32 +read_twcc_seqnum (GstBuffer * buf, guint8 twcc_ext_id) +{ + guint16 twcc_seqnum; + gpointer ext_data; + guint ext_size; + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + + if (!gst_rtp_buffer_map (buf, GST_MAP_READ, &rtp)) + return -1; + + if (!gst_rtp_buffer_get_extension_onebyte_header (&rtp, twcc_ext_id, + 0, &ext_data, &ext_size)) { + gst_rtp_buffer_unmap (&rtp); + return -1; + } + + fail_unless (ext_data != NULL); + fail_unless (ext_size == 2); + + twcc_seqnum = GST_READ_UINT16_BE (ext_data); + gst_rtp_buffer_unmap (&rtp); + + return twcc_seqnum; +} + +static GstBuffer * +generate_test_buffer_timed (GstClockTime ts, guint seqnum, guint32 rtp_ts) +{ + return generate_test_buffer_full (ts, + seqnum, rtp_ts, TEST_BUF_SSRC, FALSE, TEST_BUF_PT, 0, 0); +} + typedef struct { GstHarness *send_rtp_h; @@ -133,15 +236,23 @@ typedef struct GstTestClock *testclock; GstCaps *caps; + GHashTable *pt_to_caps_map; + gboolean running; GMutex lock; GstStructure *last_twcc_stats; } SessionHarness; static GstCaps * -_pt_map_requested (GstElement * element, guint pt, gpointer data) +_pt_map_requested (G_GNUC_UNUSED GstElement * element, guint pt, gpointer data) { SessionHarness *h = data; + GstCaps *caps = g_hash_table_lookup (h->pt_to_caps_map, + GUINT_TO_POINTER (pt)); + + if (caps) + return gst_caps_copy (caps); + return gst_caps_copy (h->caps); } @@ -161,23 +272,32 @@ _notify_twcc_stats (GParamSpec * spec G_GNUC_UNUSED, } static GstStructure * -session_harness_get_last_twcc_stats (SessionHarness * h) +session_harness_get_twcc_stats_full (SessionHarness * h, + GstClockTime stats_window_size, GstClockTime stats_window_delay) { - GstStructure *ret = NULL; - g_mutex_lock (&h->lock); - if (h->last_twcc_stats) - ret = gst_structure_copy (h->last_twcc_stats); - g_mutex_unlock (&h->lock); + GstStructure *ret; + g_signal_emit_by_name (h->internal_session, "get-twcc-windowed-stats", + stats_window_size, stats_window_delay, &ret); return ret; } +static GstStructure * +session_harness_get_twcc_stats (SessionHarness * h) +{ + return session_harness_get_twcc_stats_full (h, + 300 * GST_MSECOND, 200 * GST_MSECOND); +} + static SessionHarness * session_harness_new (void) { SessionHarness *h = g_new0 (SessionHarness, 1); - h->caps = generate_caps (); + h->caps = generate_caps (TEST_BUF_PT); g_mutex_init (&h->lock); + h->pt_to_caps_map = + g_hash_table_new_full (NULL, NULL, NULL, (GDestroyNotify) gst_caps_unref); + h->testclock = GST_TEST_CLOCK_CAST (gst_test_clock_new ()); gst_system_clock_set_default (GST_CLOCK_CAST (h->testclock)); @@ -215,6 +335,8 @@ session_harness_free (SessionHarness * h) gst_caps_unref (h->caps); gst_object_unref (h->testclock); + g_hash_table_destroy (h->pt_to_caps_map); + gst_harness_teardown (h->rtcp_h); gst_harness_teardown (h->recv_rtp_h); gst_harness_teardown (h->send_rtp_h); @@ -284,7 +406,7 @@ session_harness_advance_and_crank (SessionHarness * h, GstClockTime delta) } static void -session_harness_produce_rtcp (SessionHarness * h, gint num_rtcp_packets) +session_harness_produce_rtcp (SessionHarness * h, guint num_rtcp_packets) { /* due to randomness in rescheduling of RTCP timeout, we need to keep cranking until we have the desired amount of packets */ @@ -352,12 +474,38 @@ session_harness_set_twcc_recv_ext_id (SessionHarness * h, guint8 ext_id) g_signal_emit_by_name (h->session, "clear-pt-map"); } +static GstStructure * +create_rtx_map (const gchar * name, guint key, guint value) +{ + gchar *key_str = g_strdup_printf ("%u", key); + GstStructure *s = gst_structure_new (name, + key_str, G_TYPE_UINT, (guint) value, NULL); + g_free (key_str); + return s; +} + +static void +session_harness_enable_rtx (SessionHarness * h) +{ + GstStructure *rtx_map = + create_rtx_map ("rtx-map", TEST_BUF_SSRC, TEST_RTX_BUF_SSRC); + g_object_set (h->internal_session, "rtx-ssrc-map", rtx_map, NULL); + gst_structure_free (rtx_map); +} + +static void +session_harness_add_caps_for_pt (SessionHarness * h, GstCaps * caps, guint8 pt) +{ + g_hash_table_insert (h->pt_to_caps_map, GUINT_TO_POINTER (pt), caps); + g_signal_emit_by_name (h->session, "clear-pt-map"); +} + static void -session_harness_set_twcc_send_ext_id (SessionHarness * h, guint8 ext_id) +session_harness_add_twcc_caps_for_pt (SessionHarness * h, guint8 pt) { - GstCaps *caps = gst_caps_copy (h->caps); - _add_twcc_field_to_caps (caps, ext_id); - gst_harness_set_src_caps (h->send_rtp_h, caps); + GstCaps *caps = generate_caps (pt); + _add_twcc_field_to_caps (caps, TEST_TWCC_EXT_ID); + session_harness_add_caps_for_pt (h, caps, pt); } GST_START_TEST (test_multiple_ssrc_rr) @@ -367,7 +515,7 @@ GST_START_TEST (test_multiple_ssrc_rr) GstBuffer *in_buf, *out_buf; GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; GstRTCPPacket rtcp_packet; - gint i, j; + guint i, j; guint ssrc_match; guint ssrcs[] = { @@ -429,7 +577,7 @@ GST_START_TEST (test_multiple_senders_roundrobin_rbs) GstBuffer *buf; GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; GstRTCPPacket rtcp_packet; - gint i, j, k; + guint i, j, k; guint32 ssrc; GHashTable *rb_ssrcs, *tmp_set; @@ -672,7 +820,7 @@ GST_START_TEST (test_internal_sources_timeout) gst_buffer_unref (buf); /* ok, now let's push some RTP packets */ - caps = generate_caps (); + caps = generate_caps (TEST_BUF_PT); gst_caps_set_simple (caps, "ssrc", G_TYPE_UINT, 0x01BADBAD, "rtx-ssrc", G_TYPE_UINT, 0x01020304, NULL); @@ -788,7 +936,7 @@ typedef struct } RTCPAppResult; static void -on_app_rtcp_cb (GObject * session, guint subtype, guint ssrc, +on_app_rtcp_cb (G_GNUC_UNUSED GObject * session, guint subtype, guint ssrc, const gchar * name, GstBuffer * data, RTCPAppResult * result) { result->subtype = subtype; @@ -855,10 +1003,12 @@ GST_START_TEST (test_receive_rtcp_app_packet) GST_END_TEST; static void -stats_test_cb (GObject * object, GParamSpec * spec, gpointer data) +stats_test_cb (G_GNUC_UNUSED GObject * object, G_GNUC_UNUSED GParamSpec * spec, + gpointer data) { guint num_sources = 0; gboolean *cb_called = data; + g_assert (*cb_called == FALSE); /* We should be able to get a rtpsession property @@ -892,7 +1042,8 @@ GST_START_TEST (test_dont_lock_on_stats) GST_END_TEST; static void -suspicious_bye_cb (GObject * object, GParamSpec * spec, gpointer data) +suspicious_bye_cb (GObject * object, G_GNUC_UNUSED GParamSpec * spec, + gpointer data) { GValueArray *stats_arr; GstStructure *stats, *internal_stats; @@ -1091,7 +1242,8 @@ add_rtcp_sdes_packet (GstBuffer * gstbuf, guint32 ssrc, const char *cname) static void -on_ssrc_collision_cb (GstElement * rtpsession, guint ssrc, gpointer user_data) +on_ssrc_collision_cb (G_GNUC_UNUSED GstElement * rtpsession, + G_GNUC_UNUSED guint ssrc, gpointer user_data) { gboolean *had_collision = user_data; @@ -1321,6 +1473,102 @@ GST_START_TEST (test_ssrc_collision_third_party) GST_END_TEST; +GST_START_TEST (test_ssrc_collision_third_party_disable) +{ + SessionHarness *h = session_harness_new (); + GstBuffer *buf; + GSocketAddress *saddr; + gboolean had_collision = FALSE; + guint i; + GObject *source; + GstStructure *stats; + + g_object_set (h->internal_session, + "ssrc-collision-detection", FALSE, "favor-new", TRUE, NULL); + g_signal_connect (h->internal_session, "on-ssrc-collision", + G_CALLBACK (on_ssrc_collision_cb), &had_collision); + + for (i = 0; i < 4; i++) { + /* Receive 4 buffers SSRC=0x12345678 from 127.0.0.1 to pass probation */ + buf = generate_test_buffer (i, 0x12345678); + saddr = g_inet_socket_address_new_from_string ("127.0.0.1", 8080); + gst_buffer_add_net_address_meta (buf, saddr); + g_object_unref (saddr); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (h, buf)); + } + + /* Check that we received the first 4 buffer */ + for (i = 0; i < 4; i++) { + buf = gst_harness_pull (h->recv_rtp_h); + fail_unless (buf); + gst_buffer_unref (buf); + } + fail_unless (had_collision == FALSE); + + /* Check the source address reported by the source */ + g_signal_emit_by_name (h->internal_session, "get-source-by-ssrc", 0x12345678, + &source); + g_object_get (source, "stats", &stats, NULL); + + fail_unless_equals_string (gst_structure_get_string (stats, "rtp-from"), + "127.0.0.1:8080"); + + gst_structure_free (stats); + g_object_unref (source); + + /* Receive buffer SSRC=0x12345678 from 127.0.0.2 */ + buf = generate_test_buffer (0, 0x12345678); + saddr = g_inet_socket_address_new_from_string ("127.0.0.2", 8080); + gst_buffer_add_net_address_meta (buf, saddr); + g_object_unref (saddr); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (h, buf)); + + /* Check that we received the other buffer */ + buf = gst_harness_pull (h->recv_rtp_h); + fail_unless (buf); + gst_buffer_unref (buf); + fail_unless (had_collision == FALSE); + + /* Check the source address reported by the source has changed */ + g_signal_emit_by_name (h->internal_session, "get-source-by-ssrc", 0x12345678, + &source); + g_object_get (source, "stats", &stats, NULL); + + fail_unless_equals_string (gst_structure_get_string (stats, "rtp-from"), + "127.0.0.2:8080"); + + gst_structure_free (stats); + g_object_unref (source); + + /* Receive another buffer SSRC=0x12345678 from 127.0.0.1 */ + buf = generate_test_buffer (0, 0x12345678); + saddr = g_inet_socket_address_new_from_string ("127.0.0.1", 8080); + gst_buffer_add_net_address_meta (buf, saddr); + g_object_unref (saddr); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (h, buf)); + + + /* Check that we received the other buffer */ + buf = gst_harness_pull (h->recv_rtp_h); + fail_unless (buf); + gst_buffer_unref (buf); + fail_unless (had_collision == FALSE); + + /* Check the source address reported by the source has reverted back to the original */ + g_signal_emit_by_name (h->internal_session, "get-source-by-ssrc", 0x12345678, + &source); + g_object_get (source, "stats", &stats, NULL); + + fail_unless_equals_string (gst_structure_get_string (stats, "rtp-from"), + "127.0.0.1:8080"); + + gst_structure_free (stats); + g_object_unref (source); + + session_harness_free (h); +} + +GST_END_TEST; GST_START_TEST (test_ssrc_collision_third_party_favor_new) { @@ -1433,37 +1681,58 @@ GST_START_TEST (test_ssrc_collision_never_send_on_non_internal_source) GST_END_TEST; +static guint32 +_get_ssrc_from_event (GstEvent * event) +{ + guint ret = 0; + const GstStructure *s = gst_event_get_structure (event); + gst_structure_get_uint (s, "ssrc", &ret); + return ret; +} + GST_START_TEST (test_request_fir) { - SessionHarness *h = session_harness_new (); + SessionHarness *send_h = session_harness_new (); + SessionHarness *recv_h = session_harness_new (); GstBuffer *buf; GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; GstRTCPPacket rtcp_packet; guint8 *fci_data; + guint32 ssrc0 = 0x12345678; + guint32 ssrc1 = 0x87654321; + GstEvent *ev; /* add FIR-capabilites to our caps */ - gst_caps_set_simple (h->caps, "rtcp-fb-ccm-fir", G_TYPE_BOOLEAN, TRUE, NULL); + gst_caps_set_simple (recv_h->caps, "rtcp-fb-ccm-fir", G_TYPE_BOOLEAN, TRUE, + NULL); /* clear pt-map to removed the cached caps without fir */ - g_signal_emit_by_name (h->session, "clear-pt-map"); + g_signal_emit_by_name (recv_h->session, "clear-pt-map"); - g_object_set (h->internal_session, "internal-ssrc", 0xDEADBEEF, NULL); + g_object_set (recv_h->internal_session, "internal-ssrc", 0xDEADBEEF, NULL); - /* Receive a RTP buffer from the wire from 2 different ssrcs */ + /* "send" 2 different SSRCs from from the sender */ fail_unless_equals_int (GST_FLOW_OK, - session_harness_recv_rtp (h, generate_test_buffer (0, 0x12345678))); + session_harness_send_rtp (send_h, generate_test_buffer (0, ssrc0))); fail_unless_equals_int (GST_FLOW_OK, - session_harness_recv_rtp (h, generate_test_buffer (0, 0x87654321))); + session_harness_send_rtp (send_h, generate_test_buffer (0, ssrc1))); + + /* pull them from the sender and push them on to the receiver */ + fail_unless_equals_int (GST_FLOW_OK, + session_harness_recv_rtp (recv_h, + session_harness_pull_send_rtp (send_h))); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (recv_h, + session_harness_pull_send_rtp (send_h))); /* fix to make the test deterministic: We need to wait for the RTCP-thread to have settled to ensure the key-unit will considered once released */ - gst_test_clock_wait_for_next_pending_id (h->testclock, NULL); + gst_test_clock_wait_for_next_pending_id (recv_h->testclock, NULL); /* request FIR for both SSRCs */ - session_harness_force_key_unit (h, 0, 0x12345678, TEST_BUF_PT, NULL, NULL); - session_harness_force_key_unit (h, 0, 0x87654321, TEST_BUF_PT, NULL, NULL); + session_harness_force_key_unit (recv_h, 0, ssrc0, TEST_BUF_PT, NULL, NULL); + session_harness_force_key_unit (recv_h, 0, ssrc1, TEST_BUF_PT, NULL, NULL); - session_harness_produce_rtcp (h, 1); - buf = session_harness_pull_rtcp (h); + session_harness_produce_rtcp (recv_h, 1); + buf = session_harness_pull_rtcp (recv_h); fail_unless (gst_rtcp_buffer_validate (buf)); gst_rtcp_buffer_map (buf, GST_MAP_READ, &rtcp); @@ -1497,22 +1766,47 @@ GST_START_TEST (test_request_fir) gst_rtcp_packet_fb_get_fci_length (&rtcp_packet) * sizeof (guint32)); /* verify the FIR contains both SSRCs */ - fail_unless_equals_int (0x87654321, GST_READ_UINT32_BE (fci_data)); + fail_unless_equals_int (ssrc1, GST_READ_UINT32_BE (fci_data)); fail_unless_equals_int (1, fci_data[4]); fail_unless_equals_int (0, fci_data[5]); fail_unless_equals_int (0, fci_data[6]); fail_unless_equals_int (0, fci_data[7]); fci_data += 8; - fail_unless_equals_int (0x12345678, GST_READ_UINT32_BE (fci_data)); + fail_unless_equals_int (ssrc0, GST_READ_UINT32_BE (fci_data)); fail_unless_equals_int (1, fci_data[4]); fail_unless_equals_int (0, fci_data[5]); fail_unless_equals_int (0, fci_data[6]); fail_unless_equals_int (0, fci_data[7]); gst_rtcp_buffer_unmap (&rtcp); - gst_buffer_unref (buf); - session_harness_free (h); + + /* now "send" the produced RTCP FIR back to the sender */ + session_harness_recv_rtcp (send_h, buf); + + /* Remove the first 2 reconfigure events */ + ev = gst_harness_pull_upstream_event (send_h->send_rtp_h); + fail_unless_equals_int (GST_EVENT_RECONFIGURE, GST_EVENT_TYPE (ev)); + gst_event_unref (ev); + ev = gst_harness_pull_upstream_event (send_h->send_rtp_h); + fail_unless_equals_int (GST_EVENT_RECONFIGURE, GST_EVENT_TYPE (ev)); + gst_event_unref (ev); + + /* Then pull and check the force key-unit events, for the right SSRCs */ + ev = gst_harness_pull_upstream_event (send_h->send_rtp_h); + fail_unless_equals_int (GST_EVENT_CUSTOM_UPSTREAM, GST_EVENT_TYPE (ev)); + fail_unless (gst_video_event_is_force_key_unit (ev)); + fail_unless_equals_int (ssrc1, _get_ssrc_from_event (ev)); + gst_event_unref (ev); + + ev = gst_harness_pull_upstream_event (send_h->send_rtp_h); + fail_unless_equals_int (GST_EVENT_CUSTOM_UPSTREAM, GST_EVENT_TYPE (ev)); + fail_unless (gst_video_event_is_force_key_unit (ev)); + fail_unless_equals_int (ssrc0, _get_ssrc_from_event (ev)); + gst_event_unref (ev); + + session_harness_free (send_h); + session_harness_free (recv_h); } GST_END_TEST; @@ -1640,7 +1934,7 @@ GST_START_TEST (test_request_fir_after_pli_in_caps) /* Rebuild the caps */ gst_caps_unref (h->caps); - h->caps = generate_caps (); + h->caps = generate_caps (TEST_BUF_PT); /* add FIR-capabilites to our caps */ gst_caps_set_simple (h->caps, "rtcp-fb-ccm-fir", G_TYPE_BOOLEAN, TRUE, NULL); @@ -1729,6 +2023,37 @@ GST_START_TEST (test_illegal_rtcp_fb_packet) GST_END_TEST; +GST_START_TEST (test_illegal_rtcp_type_packet) +{ + SessionHarness *h = session_harness_new (); + GstBuffer *buf; + const guint8 rtcp_invalid_type_pkt[] = { + /* Initial SR vaid packet */ + 0x81, 0xc8, 0x00, 0x0c, 0x3f, 0x33, 0xa4, 0xed, + 0xdf, 0xfe, 0x6d, 0x48, 0xad, 0xad, 0xf4, 0x28, + 0x04, 0xce, 0x6d, 0x92, 0x00, 0x00, 0x02, 0x08, + 0x00, 0x05, 0x7b, 0x69, 0x1c, 0x71, 0x28, 0x33, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x03, 0x3b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + /* Invalid RTCP packet type (ignore it) */ + 0x80, 0x00, 0x00, 0x00 + }; + g_object_set (h->internal_session, "internal-ssrc", 0xDEADBEEF, NULL); + + buf = gst_buffer_new_and_alloc (sizeof (rtcp_invalid_type_pkt)); + gst_buffer_fill (buf, 0, rtcp_invalid_type_pkt, + sizeof (rtcp_invalid_type_pkt)); + GST_BUFFER_DTS (buf) = GST_BUFFER_PTS (buf) = G_GUINT64_CONSTANT (0); + + /* Push the packet */ + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtcp (h, buf)); + + session_harness_free (h); +} + +GST_END_TEST; + typedef struct { GCond *cond; @@ -1737,9 +2062,10 @@ typedef struct } FeedbackRTCPCallbackData; static void -feedback_rtcp_cb (GstElement * element, guint fbtype, guint fmt, - guint sender_ssrc, guint media_ssrc, GstBuffer * fci, - FeedbackRTCPCallbackData * cb_data) +feedback_rtcp_cb (G_GNUC_UNUSED GstElement * element, + G_GNUC_UNUSED guint fbtype, G_GNUC_UNUSED guint fmt, + G_GNUC_UNUSED guint sender_ssrc, G_GNUC_UNUSED guint media_ssrc, + G_GNUC_UNUSED GstBuffer * fci, FeedbackRTCPCallbackData * cb_data) { g_mutex_lock (cb_data->mutex); cb_data->fired = TRUE; @@ -2036,7 +2362,8 @@ typedef struct } BlockingProbeData; static GstPadProbeReturn -on_rtcp_pad_blocked (GstPad * pad, GstPadProbeInfo * info, gpointer user_data) +on_rtcp_pad_blocked (G_GNUC_UNUSED GstPad * pad, + G_GNUC_UNUSED GstPadProbeInfo * info, gpointer user_data) { BlockingProbeData *probe = user_data; @@ -2070,7 +2397,8 @@ session_harness_block_rtcp (SessionHarness * h, BlockingProbeData * probe) } static void -session_harness_unblock_rtcp (SessionHarness * h, BlockingProbeData * probe) +session_harness_unblock_rtcp (G_GNUC_UNUSED SessionHarness * h, + BlockingProbeData * probe) { gst_pad_remove_probe (probe->pad, probe->id); gst_object_unref (probe->pad); @@ -2322,8 +2650,9 @@ GST_START_TEST (test_disable_sr_timestamp) GST_END_TEST; static guint -on_sending_nacks (GObject * internal_session, guint sender_ssrc, - guint media_ssrc, GArray * nacks, GstBuffer * buffer) +on_sending_nacks (G_GNUC_UNUSED GObject * internal_session, + G_GNUC_UNUSED guint sender_ssrc, guint media_ssrc, GArray * nacks, + GstBuffer * buffer) { GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; GstRTCPPacket packet; @@ -2452,7 +2781,8 @@ GST_START_TEST (test_on_sending_nacks) GST_END_TEST; static void -disable_probation_on_new_ssrc (GObject * session, GObject * source) +disable_probation_on_new_ssrc (G_GNUC_UNUSED GObject * session, + GObject * source) { g_object_set (source, "probation", 0, NULL); } @@ -2615,7 +2945,7 @@ test_packet_rate_impl (gboolean stepped) SessionHarness *h = session_harness_new (); GstBuffer *buf; guint i; - const int PROBATION_CNT = 5; + const guint PROBATION_CNT = 5; GstStructure *stats; GObject *source; guint pktrate; @@ -2647,6 +2977,7 @@ test_packet_rate_impl (gboolean stepped) g_object_get (source, "stats", &stats, NULL); fail_unless (gst_structure_get_uint (stats, "recv-packet-rate", &pktrate)); + g_print ("pktrate: %u\n\n\n", pktrate); fail_unless (pktrate > 900 && pktrate < 1100); /* Allow 10% of error */ gst_structure_free (stats); @@ -2664,12 +2995,82 @@ GST_END_TEST; GST_START_TEST (test_stepped_packet_rate) { - test_packet_rate_impl (TRUE); + test_packet_rate_impl (FALSE); } GST_END_TEST; +static const guint8 pse_data[] = { + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77 +}; + +static void +_creating_srrr (G_GNUC_UNUSED GObject * session, G_GNUC_UNUSED GObject * src, + GstRTCPPacket * packet) +{ + fail_unless_equals_int (GST_RTCP_TYPE_RR, gst_rtcp_packet_get_type (packet)); + gst_rtcp_packet_add_profile_specific_ext (packet, pse_data, + sizeof (pse_data)); +} + +static void +_ssrc_pse (G_GNUC_UNUSED GObject * session, GObject * src, guint type, + GstBuffer * pse, G_GNUC_UNUSED guint rb_count, gpointer userdata) +{ + g_object_get (src, "ssrc", userdata, NULL); + + fail_unless_equals_int (GST_RTCP_TYPE_RR, type); + fail_unless_equals_int (0, + gst_buffer_memcmp (pse, 0, pse_data, sizeof (pse_data))); +} + +GST_START_TEST (test_creating_srrr) +{ + SessionHarness *h = session_harness_new (); + guint i; + GstFlowReturn res; + GstBuffer *buf; + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + GstRTCPPacket packet; + guint pse_ssrc = 0; + + /* Connect to on-creating-sr-rr which will append 8 bytes of + * profile-specific extension data */ + g_object_set (h->internal_session, "internal-ssrc", 0xDEADBEEF, NULL); + g_signal_connect (h->internal_session, "on-creating-sr-rr", + G_CALLBACK (_creating_srrr), NULL); + + /* receive some buffers */ + for (i = 0; i < 2; i++) { + buf = generate_test_buffer (i, 0x01BADBAD); + res = session_harness_recv_rtp (h, buf); + fail_unless_equals_int (GST_FLOW_OK, res); + } + + session_harness_produce_rtcp (h, 1); + buf = session_harness_pull_rtcp (h); + fail_unless (gst_rtcp_buffer_validate (buf)); + gst_rtcp_buffer_map (buf, GST_MAP_READ, &rtcp); + fail_unless (gst_rtcp_buffer_get_first_packet (&rtcp, &packet)); + fail_unless_equals_int (GST_RTCP_TYPE_RR, gst_rtcp_packet_get_type (&packet)); + fail_unless_equals_int (2, /* 2x 32bit words */ + gst_rtcp_packet_get_profile_specific_ext_length (&packet)); + gst_rtcp_buffer_unmap (&rtcp); + + /* now "receive" the same RTCP buffer into the session to test + * on-ssrc-profile-specific-ext signal */ + g_signal_connect (h->internal_session, "on-ssrc-profile-specific-ext", + G_CALLBACK (_ssrc_pse), &pse_ssrc); + fail_unless_equals_int (0, pse_ssrc); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtcp (h, buf)); + fail_unless_equals_int (0xDEADBEEF, pse_ssrc); + + session_harness_free (h); +} + +GST_END_TEST; + /********************* TWCC-tests *********************/ static GstRTCPFBType @@ -2877,48 +3278,53 @@ G_STMT_START { \ guint j = 0; \ GValueArray *packets_array = g_value_get_boxed ( \ gst_structure_get_value (gst_event_get_structure ((event)), "packets")); \ + g_assert (packets_array); \ for (i = 0; i < packets_array->n_values; i++) { \ - TWCCPacket *twcc_pkt; \ - GstClockTime ts; \ - guint seqnum; \ - gboolean lost; \ + TWCCPacket *twcc_pkt = NULL; \ + GstClockTime ts = 0; \ + guint seqnum = 0; \ + gboolean lost = FALSE; \ const GstStructure *pkt_s = \ gst_value_get_structure (g_value_array_get_nth (packets_array, i)); \ + g_assert (pkt_s); \ fail_unless (gst_structure_get_boolean (pkt_s, "lost", &lost)); \ - if (lost) \ - continue; \ - fail_unless (gst_structure_get_clock_time (pkt_s, "remote-ts", &ts)); \ - fail_unless (gst_structure_get_uint (pkt_s, "seqnum", &seqnum)); \ - twcc_pkt = &(packets)[j++]; \ - fail_unless_equals_int (twcc_pkt->seqnum, seqnum); \ - fail_unless_equals_twcc_clocktime (twcc_pkt->timestamp, ts); \ + if (!lost) { \ + fail_unless (gst_structure_get_clock_time (pkt_s, "remote-ts", &ts)); \ + fail_unless (gst_structure_get_uint (pkt_s, "seqnum", &seqnum)); \ + if (j < G_N_ELEMENTS (packets)) { \ + twcc_pkt = &(packets[j]); \ + j++; \ + } \ + if (twcc_pkt) { \ + fail_unless_equals_int (twcc_pkt->seqnum, seqnum); \ + fail_unless_equals_twcc_clocktime (twcc_pkt->timestamp, ts); \ + } \ + } \ } \ gst_event_unref (event); \ } G_STMT_END -#define twcc_verify_packets_to_packets(send_h, recv_h, packets) \ +#define twcc_verify_packets_to_packets(send_h, recv_h, recv_pkt, parsed_pkt) \ G_STMT_START { \ guint i; \ GstEvent *event; \ - twcc_push_packets ((recv_h), packets); \ + twcc_push_packets ((recv_h), recv_pkt); \ session_harness_recv_rtcp ((send_h), \ session_harness_produce_twcc ((recv_h))); \ for (i = 0; i < 2; i++) \ gst_event_unref (gst_harness_pull_upstream_event ((send_h)->send_rtp_h)); \ event = gst_harness_pull_upstream_event ((send_h)->send_rtp_h); \ - twcc_verify_packets_to_event (packets, event); \ + twcc_verify_packets_to_event (parsed_pkt, event); \ } G_STMT_END -#define twcc_verify_stats(h, bitrate_sent, bitrate_recv, pkts_sent, pkts_recv, loss_pct, avg_dod) \ +#define twcc_verify_stats(twcc_stats, bitrate_sent, bitrate_recv, pkts_sent, pkts_recv, loss_pct, avg_dod) \ G_STMT_START { \ - GstStructure *twcc_stats; \ guint stats_bitrate_sent; \ guint stats_bitrate_recv; \ guint stats_packets_sent; \ guint stats_packets_recv; \ gdouble stats_loss_pct; \ GstClockTimeDiff stats_avg_dod; \ - twcc_stats = session_harness_get_last_twcc_stats (h); \ fail_unless (gst_structure_get (twcc_stats, \ "bitrate-sent", G_TYPE_UINT, &stats_bitrate_sent, \ "bitrate-recv", G_TYPE_UINT, &stats_bitrate_recv, \ @@ -2932,7 +3338,22 @@ G_STMT_START { fail_unless_equals_int (pkts_recv, stats_packets_recv); \ fail_unless_equals_float (loss_pct, stats_loss_pct); \ fail_unless_equals_int64 (avg_dod, stats_avg_dod); \ - gst_structure_free (twcc_stats); \ +} G_STMT_END + +#define twcc_send_recv_buffers(h_send, h_recv, buffers) \ +G_STMT_START { \ + guint i; \ + session_harness_set_twcc_recv_ext_id (h_recv, TEST_TWCC_EXT_ID); \ + for (i = 0; i < G_N_ELEMENTS(buffers); i++) { \ + GstBuffer *buf = buffers[i]; \ + GstFlowReturn res = session_harness_send_rtp (h_send, buf); \ + fail_unless_equals_int (GST_FLOW_OK, res); \ + buf = session_harness_pull_send_rtp (h_send); \ + session_harness_advance_and_crank (h_send, TEST_BUF_DURATION); \ + res = session_harness_recv_rtp (h_recv, buf); \ + fail_unless_equals_int (GST_FLOW_OK, res); \ + } \ + session_harness_recv_rtcp (h_send, session_harness_produce_twcc (h_recv)); \ } G_STMT_END GST_START_TEST (test_twcc_1_bit_status_vector) @@ -2972,7 +3393,7 @@ GST_START_TEST (test_twcc_1_bit_status_vector) twcc_verify_packets_to_fci (h0, packets, exp_fci); /* and check we can parse this back to the original packets */ - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3024,7 +3445,7 @@ GST_START_TEST (test_twcc_status_vector_split_large_delta) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3060,7 +3481,7 @@ GST_START_TEST (test_twcc_2_bit_status_vector) twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3118,7 +3539,7 @@ GST_START_TEST (test_twcc_status_vector_split_with_gap) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3199,7 +3620,7 @@ GST_START_TEST (test_twcc_status_vector_split_into_three) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3232,7 +3653,7 @@ GST_START_TEST (test_twcc_2_bit_full_status_vector) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3250,7 +3671,7 @@ GST_START_TEST (test_twcc_various_gaps) {seq, seq * 250 * GST_USECOND, TRUE}, }; - twcc_verify_packets_to_packets (h, h, packets); + twcc_verify_packets_to_packets (h, h, packets, packets); session_harness_free (h); } @@ -3284,7 +3705,7 @@ GST_START_TEST (test_twcc_negative_delta) twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3318,7 +3739,7 @@ GST_START_TEST (test_twcc_seqnum_wrap) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3447,7 +3868,7 @@ GST_START_TEST (test_twcc_huge_seqnum_gap) twcc_push_packets (h0, packets); twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3493,24 +3914,117 @@ GST_START_TEST (test_twcc_duplicate_seqnums) GST_END_TEST; - -GST_START_TEST (test_twcc_multiple_markers) +GST_START_TEST (test_twcc_duplicate_previous_seqnums) { SessionHarness *h = session_harness_new (); GstBuffer *buf; - /* for this test, notice how the first recv-delta should relate back to - the reference-time, which is 0 in this case. The packets are incrementing - in timestamps equal to the smallest unit for TWCC (250 microseconds) */ + /* We receive pkt #1 again and this last one should be ignored */ TWCCPacket packets[] = { - {1, 1 * 250 * GST_USECOND, FALSE}, - {2, 2 * 250 * GST_USECOND, FALSE}, - {3, 3 * 250 * GST_USECOND, TRUE}, - {4, 4 * 250 * GST_USECOND, FALSE}, - {5, 5 * 250 * GST_USECOND, TRUE}, - {6, 6 * 250 * GST_USECOND, FALSE}, - {7, 7 * 250 * GST_USECOND, FALSE}, - {8, 8 * 250 * GST_USECOND, FALSE}, + {1, 4 * 32 * GST_MSECOND, FALSE}, + {2, 5 * 32 * GST_MSECOND, FALSE}, + {1, 6 * 32 * GST_MSECOND, FALSE}, + {3, 7 * 32 * GST_MSECOND, TRUE}, + }; + + guint8 exp_fci[] = { + 0x00, 0x01, /* base sequence number: 1 */ + 0x00, 0x03, /* packet status count: 2 */ + 0x00, 0x00, 0x02, /* reference time: 2 * 64ms */ + 0x00, /* feedback packet count: 0 */ + /* packet chunks: */ + 0xd6, 0x00, /* 1 1 0 1 0 1 1 0 | 0 0 0 0 0 0 0 0 */ + 0x00, 0x80, /* recv deltas: +0, +32ms, + 64ms */ + 0x01, 0x00, + 0x00, 0x00, /* padding */ + }; + + twcc_push_packets (h, packets); + + buf = session_harness_produce_twcc (h); + twcc_verify_fci (buf, exp_fci); + gst_buffer_unref (buf); + + session_harness_free (h); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_missing_packet_duplicates_last) +{ + SessionHarness *h = session_harness_new (); + GstBuffer *buf; + + /* Verify the behavior that if we have had a packetloss in the + previous report, we start the next report with the last packet + of the previous report */ + + /* first we have a gap of 12 packets */ + TWCCPacket packets0[] = { + {10, 0 * 250 * GST_USECOND, FALSE}, + {22, 1 * 250 * GST_USECOND, TRUE}, + }; + + /* and then two "normal" ones */ + TWCCPacket packets1[] = { + {23, 2 * 250 * GST_USECOND, FALSE}, + {24, 3 * 250 * GST_USECOND, TRUE}, + }; + + guint8 exp_fci0[] = { + 0x00, 0x0a, /* base sequence number: 10 */ + 0x00, 0x0d, /* packet status count: 13 */ + 0x00, 0x00, 0x00, /* reference time: 0 ms */ + 0x00, /* feedback packet count: 0 */ + /* packet chunks: */ + 0xa0, 0x02, /* */ + 0x00, 0x01, /* recv deltas: +0, +1 */ + }; + + guint8 exp_fci1[] = { + 0x00, 0x16, /* base sequence number: 22 */ + 0x00, 0x03, /* packet status count: 3 */ + 0x00, 0x00, 0x00, /* reference time: 0 ms */ + 0x01, /* feedback packet count: 1 */ + /* packet chunks: */ + 0x20, 0x03, /* */ + 0x01, 0x01, 0x01, /* recv deltas: +1, +1, +1 */ + 0x00, 0x00, 0x00, /* padding */ + }; + + twcc_push_packets (h, packets0); + buf = session_harness_produce_twcc (h); + twcc_verify_fci (buf, exp_fci0); + gst_buffer_unref (buf); + + twcc_push_packets (h, packets1); + buf = session_harness_produce_twcc (h); + twcc_verify_fci (buf, exp_fci1); + gst_buffer_unref (buf); + + session_harness_free (h); +} + +GST_END_TEST; + + +GST_START_TEST (test_twcc_multiple_markers) +{ + SessionHarness *h = session_harness_new (); + GstBuffer *buf; + + /* for this test, notice how the first recv-delta should relate back to + the reference-time, which is 0 in this case. The packets are incrementing + in timestamps equal to the smallest unit for TWCC (250 microseconds) */ + TWCCPacket packets[] = { + {1, 1 * 250 * GST_USECOND, FALSE}, + {2, 2 * 250 * GST_USECOND, FALSE}, + {3, 3 * 250 * GST_USECOND, TRUE}, + {4, 4 * 250 * GST_USECOND, FALSE}, + {5, 5 * 250 * GST_USECOND, TRUE}, + {6, 6 * 250 * GST_USECOND, FALSE}, + {7, 7 * 250 * GST_USECOND, FALSE}, + {8, 8 * 250 * GST_USECOND, FALSE}, {9, 9 * 250 * GST_USECOND, TRUE}, }; @@ -3768,8 +4282,7 @@ GST_START_TEST (test_twcc_double_gap) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -3780,59 +4293,255 @@ GST_END_TEST; GST_START_TEST (test_twcc_recv_packets_reordered) { SessionHarness *h = session_harness_new (); - GstBuffer *buf; - /* a reordered seqence, with marker-bits for #3 and #4 */ - TWCCPacket packets[] = { - {1, 1 * 250 * GST_USECOND, FALSE} - , - {3, 2 * 250 * GST_USECOND, TRUE} - , - {2, 3 * 250 * GST_USECOND, FALSE} - , - {4, 4 * 250 * GST_USECOND, TRUE} - , + /* *INDENT-OFF* */ + TWCCPacket packets0[] = { + { 1, 1 * 250 * GST_USECOND, FALSE}, + { 2, 2 * 250 * GST_USECOND, FALSE}, + { 3, 3 * 250 * GST_USECOND, FALSE}, + { 4, 4 * 250 * GST_USECOND, FALSE}, + {10, 5 * 250 * GST_USECOND, TRUE}, + }; + + TWCCPacket packets1[] = { + { 5, 6 * 250 * GST_USECOND, FALSE}, + { 6, 7 * 250 * GST_USECOND, FALSE}, + { 7, 8 * 250 * GST_USECOND, FALSE}, + { 8, 9 * 250 * GST_USECOND, TRUE}, + }; + + TWCCPacket packets2[] = { + { 9, 10 * 250 * GST_USECOND, FALSE}, + { 11, 11 * 250 * GST_USECOND, TRUE}, }; + /* *INDENT-ON* */ - /* first we expect #2 to be reported lost */ + /* this reports 1 to 4, then 5 to 9 missing, and then 10 */ guint8 exp_fci0[] = { 0x00, 0x01, /* base sequence number: 1 */ - 0x00, 0x03, /* packet status count: 3 */ + 0x00, 0x0a, /* packet status count: 10 */ 0x00, 0x00, 0x00, /* reference time: 0 */ 0x00, /* feedback packet count: 0 */ /* packet chunks: */ - 0xa8, 0x00, /* 1 0 1 0 1 0 0 0 | 0 0 0 0 0 0 0 0 */ - 0x01, 0x01, /* recv deltas, +1, +1 */ + 0xbc, 0x10, + 0x01, 0x01, 0x01, 0x01, 0x01, + 0x00, /* padding */ }; - /* and then when 2 actually arrives, it is already reported lost, - so we will not re-report it, but drop it */ + /* this reports 5 to 8, 9 missing and then 10 (again) */ guint8 exp_fci1[] = { - 0x00, 0x04, /* base sequence number: 4 */ - 0x00, 0x01, /* packet status count: 1 */ + 0x00, 0x05, /* base sequence number: 5 */ + 0x00, 0x06, /* packet status count: 6 */ 0x00, 0x00, 0x00, /* reference time: 0 */ 0x01, /* feedback packet count: 1 */ /* packet chunks: */ - 0x20, 0x01, /* 0 0 1 0 0 0 0 0 | 0 0 0 0 0 0 0 1 */ - 0x04, /* recv deltas, +4 */ + 0xd5, 0x48, + 0x06, + 0x01, 0x01, 0x01, + 0xff, 0xfc, + }; + + /* this reports 9, 10 (again!) and 11 */ + guint8 exp_fci2[] = { + 0x00, 0x09, /* base sequence number: 4 */ + 0x00, 0x03, /* packet status count: 3 */ + 0x00, 0x00, 0x00, /* reference time: 0 */ + 0x02, /* feedback packet count: 2 */ + /* packet chunks: */ + 0xd9, 0x00, + 0x0a, + 0xff, 0xfb, + 0x06, + 0x00, 0x00, /* padding */ + }; + + twcc_verify_packets_to_fci (h, packets0, exp_fci0); + twcc_verify_packets_to_fci (h, packets1, exp_fci1); + twcc_verify_packets_to_fci (h, packets2, exp_fci2); + + session_harness_free (h); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_recv_packets_reordered_and_lost) +{ + SessionHarness *h0 = session_harness_new (); + SessionHarness *h1 = session_harness_new (); + + /* *INDENT-OFF* */ + TWCCPacket packets[] = { + { 1488, 557500000, FALSE}, + { 1494, 558500000, FALSE}, + { 1503, 555250000, FALSE}, + { 1504, 559500000, FALSE}, + { 1509, 595500000, TRUE}, + }; + /* *INDENT-ON* */ + + /* this reports 1 to 4, then 5 to 9 missing, and then 10 */ + guint8 exp_fci[] = { + 0x05, 0xd0, /* base sequence number: 1488 */ + 0x00, 0x16, /* packet status count: 22 */ + 0x00, 0x00, 0x08, /* reference time: 0 */ + 0x00, /* feedback packet count: 0 */ + /* packet chunks: */ + 0xd0, 0x01, /* 11 01 00 00 | 00 00 00 01 - packet 1488, then 5 missing, then 1494 */ + 0x00, 0x08, /* 00 00 00 00 | 00 00 10 00 - run length of 8 missing packets, 1495 to 1502 */ + 0xe4, 0x01, /* 11 10 01 00 | 00 00 00 01 - packets 1503, 1504 and 1509 */ + + 0xb6, 0x04, 0xff, 0xf3, 0x11, 0x90, + }; + + twcc_verify_packets_to_fci (h0, packets, exp_fci); + twcc_verify_packets_to_packets (h1, h1, packets, packets); + + session_harness_free (h0); + session_harness_free (h1); +} + +GST_END_TEST; + + +GST_START_TEST (test_twcc_recv_packets_reordered_within_report_interval) +{ + SessionHarness *h0 = session_harness_new (); + SessionHarness *h1 = session_harness_new (); + + /* a reordered seqence, all within the same report */ + TWCCPacket packets[] = { + {3, 250 * GST_USECOND, FALSE}, + {1, 500 * GST_USECOND, FALSE}, + {2, 750 * GST_USECOND, FALSE}, + {4, 1000 * GST_USECOND, TRUE}, + }; + + TWCCPacket reorderd_packets[] = { + {1, 500 * GST_USECOND, FALSE}, + {2, 750 * GST_USECOND, FALSE}, + {3, 250 * GST_USECOND, FALSE}, + {4, 1000 * GST_USECOND, TRUE}, + }; + + /* we expect this to be handled gracefully */ + guint8 exp_fci[] = { + 0x00, 0x01, /* base sequence number: 1 */ + 0x00, 0x04, /* packet status count: 3 */ + 0x00, 0x00, 0x00, /* reference time: 0 */ + 0x00, /* feedback packet count: 0 */ + /* packet chunks: */ + 0xd6, 0x40, /* 11 - Status Vector Chunk, 2 bit + 01 - Packet received, small delta + 01 - Packet received, small delta + 10 - Packet received, large or negative delta + 01 - Packet received, small delta + */ + 0x02, /* + 500us - abs: 500us */ + 0x01, /* + 250us - abs: 750us */ + 0xff, 0xfe, /* - 500us - abs: 250us */ + 0x03, /* + 750us - abs: 1000us */ 0x00, /* padding */ }; - twcc_push_packets (h, packets); + twcc_verify_packets_to_fci (h0, packets, exp_fci); + twcc_verify_packets_to_packets (h1, h1, packets, reorderd_packets); - buf = session_harness_produce_twcc (h); - twcc_verify_fci (buf, exp_fci0); - gst_buffer_unref (buf); + session_harness_free (h0); + session_harness_free (h1); +} - buf = session_harness_produce_twcc (h); - twcc_verify_fci (buf, exp_fci1); - gst_buffer_unref (buf); +GST_END_TEST; - session_harness_free (h); +GST_START_TEST (test_twcc_reordering_send_recv) +{ + SessionHarness *h_send = session_harness_new (); + SessionHarness *h_recv = session_harness_new (); + GList *bufs = NULL; + GstBuffer *buf; + GstStructure *twcc_stats; + + g_object_set (h_recv->internal_session, "twcc-feedback-interval", + 50 * GST_MSECOND, NULL); + + /* enable twcc */ + session_harness_set_twcc_recv_ext_id (h_recv, TEST_TWCC_EXT_ID); + session_harness_add_twcc_caps_for_pt (h_send, TEST_BUF_PT); + + /* sender sends the buffers */ + for (guint i = 0; i < 12; i++) { + fail_unless_equals_int (GST_FLOW_OK, session_harness_send_rtp (h_send, + generate_twcc_send_buffer (i, FALSE))); + session_harness_advance_and_crank (h_send, TEST_BUF_DURATION); + /* get the buffer ready for the network */ + bufs = g_list_append (bufs, session_harness_pull_send_rtp (h_send)); + } + + /* reorder packet 10 to the 5th position */ + buf = g_list_nth_data (bufs, 10); + bufs = g_list_remove (bufs, buf); + bufs = g_list_insert (bufs, buf, 5); + + /* 1 - send packets 0,1,2,3,4,10(!) */ + for (guint i = 0; i < 6; i++) { + buf = g_list_nth_data (bufs, i); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (h_recv, + buf)); + } + buf = session_harness_produce_twcc (h_recv); + session_harness_recv_rtcp (h_send, buf); + twcc_stats = session_harness_get_twcc_stats (h_send); + twcc_verify_stats (twcc_stats, 0, 0, 0, 0, 0.0f, 0); + gst_structure_free (twcc_stats); + + /* 2 - send packets 5,6,7,8 */ + for (guint i = 6; i < 10; i++) { + buf = g_list_nth_data (bufs, i); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (h_recv, + buf)); + } + buf = session_harness_produce_twcc (h_recv); + session_harness_recv_rtcp (h_send, buf); + twcc_stats = session_harness_get_twcc_stats (h_send); + twcc_verify_stats (twcc_stats, 0, 0, 0, 0, 0.0f, 0); + gst_structure_free (twcc_stats); + + /* 3 - send packets 9,11 */ + for (guint i = 10; i < 12; i++) { + buf = g_list_nth_data (bufs, i); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (h_recv, + buf)); + } + buf = session_harness_produce_twcc (h_recv); + session_harness_recv_rtcp (h_send, buf); + twcc_stats = session_harness_get_twcc_stats (h_send); + twcc_verify_stats (twcc_stats, 532800, 532800, 2, 2, 0.0f, 0); + gst_structure_free (twcc_stats); + + /* now send enough buffers to get within our stats window */ + for (guint i = 12; i < 26; i++) { + fail_unless_equals_int (GST_FLOW_OK, session_harness_send_rtp (h_send, + generate_twcc_send_buffer (i, FALSE))); + session_harness_advance_and_crank (h_send, TEST_BUF_DURATION); + buf = session_harness_pull_send_rtp (h_send); + session_harness_recv_rtp (h_recv, buf); + } + buf = session_harness_produce_twcc (h_recv); + session_harness_recv_rtcp (h_send, buf); + + /* we get stats for the first 16 packets, noticing no loss + even though packets were heavily reordered */ + twcc_stats = session_harness_get_twcc_stats (h_send); + twcc_verify_stats (twcc_stats, 532800, 532800, 16, 16, 0.0f, 0); + gst_structure_free (twcc_stats); + + g_list_free (bufs); + session_harness_free (h_send); + session_harness_free (h_recv); } GST_END_TEST; + GST_START_TEST (test_twcc_recv_late_packet_fb_pkt_count_wrap) { SessionHarness *h = session_harness_new (); @@ -3851,13 +4560,13 @@ GST_START_TEST (test_twcc_recv_late_packet_fb_pkt_count_wrap) }; guint8 exp_fci1[] = { - 0x01, 0x01, /* base sequence number: 257 */ + 0x00, 0xff, /* base sequence number: 255 */ 0x00, 0x01, /* packet status count: 1 */ - 0x00, 0x00, 0x01, /* reference time: 1 */ + 0x00, 0x00, 0x00, /* reference time: 0 */ 0x01, /* feedback packet count: 1 */ /* packet chunks: */ 0x20, 0x01, /* 0 0 1 0 0 0 0 0 | 0 0 0 0 0 0 0 1 */ - 0x01, /* 1 recv-delta */ + 0xff, /* 1 recv-delta */ 0x00, /* padding */ }; @@ -3918,32 +4627,27 @@ GST_START_TEST (test_twcc_recv_rtcp_reordered) guint i; /* three frames, two packets each */ - TWCCPacket packets[] = { - {1, 1 * GST_SECOND, FALSE} - , - {2, 2 * GST_SECOND, TRUE} - , - {3, 3 * GST_SECOND, FALSE} - , - {4, 4 * GST_SECOND, TRUE} - , - {5, 5 * GST_SECOND, FALSE} - , - {6, 6 * GST_SECOND, TRUE} - , - {7, 7 * GST_SECOND, FALSE} - , - {8, 8 * GST_SECOND, TRUE} - , + TWCCPacket packets0[] = { + {1, 1 * GST_SECOND, FALSE}, + {2, 2 * GST_SECOND, TRUE}, }; - -/* - TWCCPacket expected_packets0[] = { - {1, 1 * 250 * GST_USECOND, FALSE}, - {2, 2 * 250 * GST_USECOND, TRUE}, + TWCCPacket packets1[] = { + {3, 3 * GST_SECOND, FALSE}, + {4, 4 * GST_SECOND, TRUE}, + }; + TWCCPacket packets2[] = { + {5, 5 * GST_SECOND, FALSE}, + {6, 6 * GST_SECOND, TRUE}, + }; + TWCCPacket packets3[] = { + {7, 7 * GST_SECOND, FALSE}, + {8, 8 * GST_SECOND, TRUE}, }; -*/ - twcc_push_packets (recv_h, packets); + + twcc_push_packets (recv_h, packets0); + twcc_push_packets (recv_h, packets1); + twcc_push_packets (recv_h, packets2); + twcc_push_packets (recv_h, packets3); buf[0] = session_harness_produce_twcc (recv_h); buf[1] = session_harness_produce_twcc (recv_h); @@ -3960,16 +4664,16 @@ GST_START_TEST (test_twcc_recv_rtcp_reordered) gst_event_unref (gst_harness_pull_upstream_event (send_h->send_rtp_h)); event = gst_harness_pull_upstream_event (send_h->send_rtp_h); - twcc_verify_packets_to_event (&packets[0 * 2], event); + twcc_verify_packets_to_event (packets0, event); event = gst_harness_pull_upstream_event (send_h->send_rtp_h); - twcc_verify_packets_to_event (&packets[2 * 2], event); + twcc_verify_packets_to_event (packets2, event); event = gst_harness_pull_upstream_event (send_h->send_rtp_h); - twcc_verify_packets_to_event (&packets[1 * 2], event); + twcc_verify_packets_to_event (packets1, event); event = gst_harness_pull_upstream_event (send_h->send_rtp_h); - twcc_verify_packets_to_event (&packets[3 * 2], event); + twcc_verify_packets_to_event (packets3, event); session_harness_free (send_h); session_harness_free (recv_h); @@ -4000,7 +4704,7 @@ GST_START_TEST (test_twcc_send_and_recv) /* enable twcc */ session_harness_set_twcc_recv_ext_id (h_recv, TEST_TWCC_EXT_ID); - session_harness_set_twcc_send_ext_id (h_send, TEST_TWCC_EXT_ID); + session_harness_add_twcc_caps_for_pt (h_send, TEST_BUF_PT); for (frame = 0; frame < num_frames; frame++) { GstBuffer *buf; @@ -4030,9 +4734,12 @@ GST_START_TEST (test_twcc_send_and_recv) /* sender receives the TWCC packet */ session_harness_recv_rtcp (h_send, buf); - if (frame > 0) - twcc_verify_stats (h_send, TEST_BUF_BPS, TEST_BUF_BPS, num_slices, - num_slices, 0.0f, 0); + if (frame > 0) { + GstStructure *twcc_stats = session_harness_get_twcc_stats (h_send); + twcc_verify_stats (twcc_stats, 532800, 532800, num_slices + 1, + num_slices + 1, 0.0f, 0); + gst_structure_free (twcc_stats); + } } session_harness_free (h_send); @@ -4045,8 +4752,7 @@ GST_START_TEST (test_twcc_multiple_payloads_below_window) { SessionHarness *h_send = session_harness_new (); SessionHarness *h_recv = session_harness_new (); - - guint i; + GstStructure *twcc_stats; GstBuffer *buffers[] = { generate_twcc_send_buffer_full (0, FALSE, 0xabc, 98), @@ -4057,28 +4763,13 @@ GST_START_TEST (test_twcc_multiple_payloads_below_window) }; /* enable twcc */ - session_harness_set_twcc_recv_ext_id (h_recv, TEST_TWCC_EXT_ID); - session_harness_set_twcc_send_ext_id (h_send, TEST_TWCC_EXT_ID); - - for (i = 0; i < G_N_ELEMENTS (buffers); i++) { - GstBuffer *buf = buffers[i]; - GstFlowReturn res; - - /* from payloder to rtpbin */ - res = session_harness_send_rtp (h_send, buf); - fail_unless_equals_int (GST_FLOW_OK, res); - - buf = session_harness_pull_send_rtp (h_send); - session_harness_advance_and_crank (h_send, TEST_BUF_DURATION); - - /* buffer arrives at the receiver */ - res = session_harness_recv_rtp (h_recv, buf); - fail_unless_equals_int (GST_FLOW_OK, res); - } + session_harness_add_twcc_caps_for_pt (h_send, 98); + session_harness_add_twcc_caps_for_pt (h_send, 111); - /* sender receives the TWCC packet from the receiver */ - session_harness_recv_rtcp (h_send, session_harness_produce_twcc (h_recv)); - twcc_verify_stats (h_send, 0, 0, 5, 5, 0.0f, GST_CLOCK_STIME_NONE); + twcc_send_recv_buffers (h_send, h_recv, buffers); + twcc_stats = session_harness_get_twcc_stats (h_send); + twcc_verify_stats (twcc_stats, 0, 0, 0, 0, 0.0f, 0); + gst_structure_free (twcc_stats); session_harness_free (h_send); session_harness_free (h_recv); @@ -4097,7 +4788,7 @@ typedef struct static TWCCFeedbackIntervalCtx test_twcc_feedback_interval_ctx[] = { {50 * GST_MSECOND, 21, 10 * GST_MSECOND, 4}, {50 * GST_MSECOND, 16, 7 * GST_MSECOND, 2}, - {50 * GST_MSECOND, 16, 66 * GST_MSECOND, 15}, + {50 * GST_MSECOND, 16, 66 * GST_MSECOND, 16}, {50 * GST_MSECOND, 15, 33 * GST_MSECOND, 9}, }; @@ -4106,23 +4797,78 @@ GST_START_TEST (test_twcc_feedback_interval) SessionHarness *h = session_harness_new (); GstBuffer *buf; TWCCFeedbackIntervalCtx *ctx = &test_twcc_feedback_interval_ctx[__i__]; + GstClockTime ts, next_feedback_time, last_twcc_time, inter_arrival_sum; + GstClockTime expected_inter_arrival_sum; + guint feedback_received = 0; session_harness_set_twcc_recv_ext_id (h, TEST_TWCC_EXT_ID); g_object_set (h->internal_session, "twcc-feedback-interval", ctx->interval, NULL); + ts = gst_clock_get_time (GST_CLOCK_CAST (h->testclock)); + next_feedback_time = ts + ctx->interval; + last_twcc_time = GST_CLOCK_TIME_NONE; + inter_arrival_sum = 0; + for (guint i = 0; i < ctx->num_packets; i++) { - GstClockTime ts = i * ctx->ts_delta; - gst_test_clock_set_time ((h->testclock), ts); + /* Advance to last TWCC interval before ts */ + while (next_feedback_time < ts) { + session_harness_crank_clock (h); + gst_test_clock_wait_for_next_pending_id (h->testclock, NULL); + next_feedback_time += ctx->interval; + } + + /* Advance time, if we haven't already gone past it */ + if (ts > gst_clock_get_time (GST_CLOCK_CAST (h->testclock))) + gst_test_clock_set_time ((h->testclock), ts); + + /* Push recv RTP */ fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtp (h, generate_twcc_recv_buffer (i, ts, FALSE))); + + if (next_feedback_time <= ts + ctx->ts_delta) { + GstClockTime now; + + /* We expect a feedback report */ + buf = session_harness_produce_twcc (h); + gst_buffer_unref (buf); + + /* Time will have advanced to the feedback send time */ + now = gst_clock_get_time (GST_CLOCK_CAST (h->testclock)); + if (GST_CLOCK_TIME_IS_VALID (last_twcc_time)) + inter_arrival_sum += (now - last_twcc_time); + last_twcc_time = now; + feedback_received += 1; + + /* Compute next expected feedback time */ + next_feedback_time += ctx->interval; + } + + ts += ctx->ts_delta; } - for (guint i = 0; i < ctx->num_feedback; i++) { - buf = session_harness_produce_twcc (h); - gst_buffer_unref (buf); + /* Compute expected inter-arrival sum for feedback reports */ + if (ctx->ts_delta <= ctx->interval) { + /* Easy case: delta between packets is less than the feedback interval. + * In this case we expect the feedback reports to be continuous and + * spaced at the specified interval + */ + expected_inter_arrival_sum = (ctx->num_feedback - 1) * ctx->interval; + } else { + /* Inter-packet delta is more than the feedback interval. + * In this case we expect gaps in the feedback stream (because we do + * not send empty feedback reports) and thus the sum of deltas between + * feedback reports must be equal to the next multiple of the feedback + * interval after the time at which the last packet is sent + */ + expected_inter_arrival_sum = ((ctx->num_feedback - 1) * ctx->ts_delta) / + ctx->interval * ctx->interval; } + /* Ensure we got the reports we expected, spaced correctly */ + g_assert_cmpint (feedback_received, ==, ctx->num_feedback); + g_assert_cmpint (inter_arrival_sum, ==, expected_inter_arrival_sum); + session_harness_free (h); } @@ -4233,25 +4979,282 @@ GST_START_TEST (test_twcc_feedback_old_seqnum) GST_END_TEST; -GST_START_TEST (test_twcc_run_length_max) +static guint +construct_initial_state_for_rtx (SessionHarness * h_send, + SessionHarness * h_recv) { - SessionHarness *h0 = session_harness_new (); - SessionHarness *h1 = session_harness_new (); + guint i; + guint window_size_ms = 300; + guint num_buffers = window_size_ms / TEST_BUF_MS + 1; - TWCCPacket packets[] = { - /* *INDENT-OFF* */ - {0, 1000 * GST_USECOND, FALSE}, - {8205, 2000 * GST_USECOND, TRUE}, - /* *INDENT-ON* */ - }; + session_harness_enable_rtx (h_send); + session_harness_enable_rtx (h_recv); - guint8 exp_fci[] = { - 0x00, 0x00, /* base sequence number: 0 */ - 0x20, 0x0e, /* packet status count: 8206 */ - 0x00, 0x00, 0x00, /* reference time: 0 */ - 0x00, /* feedback packet count: 0 */ + /* send and recv enough packets to be over the stats window */ + for (i = 0; i < num_buffers; i++) { + GstFlowReturn ret; + GstBuffer *buf; + gboolean is_last = (i == num_buffers - 1); - 0xa0, 0x00, /* 1bit status for #0 received: 1 0 1 0 0 0 0 0 | 0 0 0 0 0 0 0 0 */ + buf = generate_twcc_send_buffer (i, is_last); + ret = session_harness_send_rtp (h_send, buf); + fail_unless_equals_int (ret, GST_FLOW_OK); + session_harness_advance_and_crank (h_send, TEST_BUF_DURATION); + + buf = session_harness_pull_send_rtp (h_send); + ret = session_harness_recv_rtp (h_recv, buf); + fail_unless_equals_int (ret, GST_FLOW_OK); + session_harness_advance_and_crank (h_recv, TEST_BUF_DURATION); + } + + /* produce a twcc feedback to process those packets */ + session_harness_recv_rtcp (h_send, session_harness_produce_twcc (h_recv)); + + return i; +} + +static void +fail_unless_twcc_stats_recovery (SessionHarness * h, gdouble recovery_pct) +{ + gdouble stats_recovery_pct; + GstStructure *twcc_stats; + + twcc_stats = session_harness_get_twcc_stats_full (h, + 300 * GST_MSECOND, 100 * GST_MSECOND); + + fail_unless (gst_structure_get (twcc_stats, + "recovery-pct", G_TYPE_DOUBLE, &stats_recovery_pct, NULL)); + fail_unless_equals_float (recovery_pct, stats_recovery_pct); + + gst_structure_free (twcc_stats); +} + +static void +send_recv_buffer (SessionHarness * h_send, SessionHarness * h_recv, + GstBuffer * buf, gboolean recv_buf) +{ + GstFlowReturn ret; + + ret = session_harness_send_rtp (h_send, buf); + fail_unless_equals_int64 (ret, GST_FLOW_OK); + + session_harness_advance_and_crank (h_send, TEST_BUF_DURATION); + + if (recv_buf) { + ret = session_harness_recv_rtp (h_recv, + session_harness_pull_send_rtp (h_send)); + fail_unless_equals_int64 (ret, GST_FLOW_OK); + session_harness_advance_and_crank (h_recv, TEST_BUF_DURATION); + } else { + /* discard that buffer from the sender */ + gst_buffer_unref (session_harness_pull_send_rtp (h_send)); + } + +} + +static void +test_twcc_stats_rtx_recovery (gboolean rtx_arrive, gdouble recovery_pct) +{ + SessionHarness *h_send = session_harness_new (); + SessionHarness *h_recv = session_harness_new (); + guint i, next_seqnum; + + session_harness_add_twcc_caps_for_pt (h_send, TEST_BUF_PT); + session_harness_add_twcc_caps_for_pt (h_send, TEST_RTX_BUF_PT); + session_harness_set_twcc_recv_ext_id (h_recv, TEST_TWCC_EXT_ID); + + next_seqnum = construct_initial_state_for_rtx (h_send, h_recv); + + for (i = 0; i < 3; i++) { + GstBuffer *buf; + GstBuffer *rtx_buf; + + buf = generate_twcc_send_buffer (next_seqnum++, FALSE); + rtx_buf = generate_rtx_buffer (i, buf); + + /* we send a buffer but receiver doesn't get it */ + send_recv_buffer (h_send, h_recv, buf, FALSE); + send_recv_buffer (h_send, h_recv, rtx_buf, rtx_arrive); + } + + /* push a last buffer with the marker bit to trigger the report */ + send_recv_buffer (h_send, h_recv, + generate_twcc_send_buffer (next_seqnum++, TRUE), TRUE); + + fail_unless_equals_int64 (GST_FLOW_OK, + session_harness_recv_rtcp (h_send, + session_harness_produce_twcc (h_recv))); + + fail_unless_twcc_stats_recovery (h_send, recovery_pct); + + session_harness_free (h_send); + session_harness_free (h_recv); +} + +GST_START_TEST (test_twcc_stats_rtx_recover_lost) +{ + test_twcc_stats_rtx_recovery (TRUE, 100.0); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_stats_no_rtx_no_recover) +{ + test_twcc_stats_rtx_recovery (FALSE, 0.0); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_feedback_max_sent_packets) +{ + /* test is very intense for valgrind or debug build */ + if (RUNNING_ON_VALGRIND || g_getenv ("EXECUTING_UNDER_DEBUG_BUILD")) + return; + + SessionHarness *h = session_harness_new (); + GstStructure *twcc_stats; + guint i; + + guint8 fci[] = { + 0x00, 0x00, /* base sequence number: 0 */ + 0xff, 0xff, /* packet status count: 65535 */ + 0x00, 0x00, 0x00, /* reference time: 0 */ + 0x00, /* feedback packet count: 0 */ + 0x1f, 0xff, /* (9x) run-length with max length as not recv: 0 0 0 1 1 1 1 1 | 1 1 1 1 1 1 1 1 */ + 0x1f, 0xff, + 0x1f, 0xff, + 0x1f, 0xff, + 0x1f, 0xff, + 0x1f, 0xff, + 0x1f, 0xff, + 0x1f, 0xff, + 0x1f, 0xff, + }; + + session_harness_add_twcc_caps_for_pt (h, TEST_BUF_PT); + + /* send over 65536 packets */ + for (i = 0; i < 65536 + 65535; i++) { + session_harness_send_rtp (h, generate_twcc_send_buffer (i, FALSE)); + session_harness_advance_and_crank (h, TEST_BUF_DURATION); + } + + /* receive the feedback message and verify the packets in our window */ + session_harness_recv_rtcp (h, + generate_twcc_feedback_rtcp (fci, sizeof (fci))); + twcc_stats = session_harness_get_twcc_stats_full (h, + 1000 * GST_MSECOND, 40 * GST_MSECOND); + twcc_verify_stats (twcc_stats, 532800, 0, 51, 0, 100.0f, 0); + gst_structure_free (twcc_stats); + + session_harness_free (h); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_non_twcc_pkts_does_not_mark_loss) +{ + SessionHarness *h_send = session_harness_new (); + SessionHarness *h_recv = session_harness_new (); + GstStructure *twcc_stats; + + /* *INDENT-OFF* */ + GstBuffer *buffers[] = { + generate_twcc_send_buffer_full (0, FALSE, 0xabc, 98), + generate_twcc_send_buffer_full (0, FALSE, 0xabc, 99), + generate_twcc_send_buffer_full (1, FALSE, 0xabc, 99), + generate_twcc_send_buffer_full (1, TRUE, 0xabc, 98), + generate_twcc_send_buffer_full (2, TRUE, 0xabc, 99), + }; + /* *INDENT-ON* */ + + session_harness_add_twcc_caps_for_pt (h_send, 99); + + twcc_send_recv_buffers (h_send, h_recv, buffers); + twcc_stats = session_harness_get_twcc_stats (h_send); + twcc_verify_stats (twcc_stats, 0, 0, 0, 0, 0.0f, 0); + gst_structure_free (twcc_stats); + + session_harness_free (h_send); + session_harness_free (h_recv); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_non_twcc_pt_no_twcc_seqnum) +{ + SessionHarness *h; + GstBuffer *buf; + + h = session_harness_new (); + session_harness_add_twcc_caps_for_pt (h, 99); + + /* push a buffer with a pt marked for twcc */ + buf = generate_twcc_send_buffer_full (0, FALSE, 0xabc, 99); + fail_unless_equals_int64 (session_harness_send_rtp (h, buf), GST_FLOW_OK); + + /* expect the twcc-seqnum to be written */ + buf = session_harness_pull_send_rtp (h); + fail_unless (read_twcc_seqnum (buf, TEST_TWCC_EXT_ID) == 0); + gst_buffer_unref (buf); + + /* push a buffer with a different pt */ + buf = generate_twcc_send_buffer_full (0, FALSE, 0xabc, 98); + fail_unless_equals_int64 (session_harness_send_rtp (h, buf), GST_FLOW_OK); + + /* expect no twcc-seqnum */ + buf = session_harness_pull_send_rtp (h); + fail_unless (read_twcc_seqnum (buf, TEST_TWCC_EXT_ID) == -1); + gst_buffer_unref (buf); + + session_harness_free (h); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_overwrites_exthdr_seqnum_if_present) +{ + SessionHarness *h; + GstBuffer *buf; + + h = session_harness_new (); + session_harness_add_twcc_caps_for_pt (h, TEST_BUF_PT); + + /* sending the ext-id in generate_test_buffer_full() will add the one-byte + header extension */ + buf = + generate_test_buffer_full (0, 0, 0, 0xabc, FALSE, TEST_BUF_PT, + TEST_TWCC_EXT_ID, 255); + fail_unless_equals_int64 (session_harness_send_rtp (h, buf), GST_FLOW_OK); + + buf = session_harness_pull_send_rtp (h); + fail_unless (read_twcc_seqnum (buf, TEST_TWCC_EXT_ID) != 255); + gst_buffer_unref (buf); + + session_harness_free (h); +} + +GST_END_TEST; + +GST_START_TEST (test_twcc_run_length_max) +{ + SessionHarness *h0 = session_harness_new (); + SessionHarness *h1 = session_harness_new (); + + TWCCPacket packets[] = { + /* *INDENT-OFF* */ + { 0, 1000 * GST_USECOND, FALSE }, + { 8205, 2000 * GST_USECOND, TRUE }, + /* *INDENT-ON* */ + }; + + guint8 exp_fci[] = { + 0x00, 0x00, /* base sequence number: 0 */ + 0x20, 0x0e, /* packet status count: 8206 */ + 0x00, 0x00, 0x00, /* reference time: 0 */ + 0x00, /* feedback packet count: 0 */ + + 0xa0, 0x00, /* 1bit status for #0 received: 1 0 1 0 0 0 0 0 | 0 0 0 0 0 0 0 0 */ 0x1f, 0xff, /* run-length with max length is reported as not received: 0 0 0 1 1 1 1 1 | 1 1 1 1 1 1 1 1 */ 0xa0, 0x00, /* 1bit status for #8205 received: 1 0 1 0 0 0 0 0 | 0 0 0 0 0 0 0 0 */ @@ -4260,7 +5263,7 @@ GST_START_TEST (test_twcc_run_length_max) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -4275,8 +5278,8 @@ GST_START_TEST (test_twcc_run_length_min) TWCCPacket packets[] = { /* *INDENT-OFF* */ - {0, 1000 * GST_USECOND, FALSE}, - {29, 2000 * GST_USECOND, TRUE}, + { 0, 1000 * GST_USECOND, FALSE }, + { 29, 2000 * GST_USECOND, TRUE }, /* *INDENT-ON* */ }; @@ -4295,7 +5298,7 @@ GST_START_TEST (test_twcc_run_length_min) }; twcc_verify_packets_to_fci (h0, packets, exp_fci); - twcc_verify_packets_to_packets (h1, h1, packets); + twcc_verify_packets_to_packets (h1, h1, packets, packets); session_harness_free (h0); session_harness_free (h1); @@ -4303,6 +5306,547 @@ GST_START_TEST (test_twcc_run_length_min) GST_END_TEST; +static void +set_min_rtcp_interval_on_new_sender_ssrc (GObject * session, + G_GNUC_UNUSED GObject * source) +{ + /* The value is irrelevant here: we want the side-effect of + * causing next_rtcp_check_time to be modified */ + g_object_set (session, "rtcp-min-interval", 5 * GST_SECOND, NULL); +} + +GST_START_TEST (test_twcc_feedback_interval_new_internal_source) +{ + SessionHarness *h = session_harness_new (); + GstBuffer *buf; + + g_object_set (h->internal_session, + "twcc-feedback-interval", 50 * GST_MSECOND, NULL); + + g_object_set (h->internal_session, "internal-ssrc", 0xDEADBEEF, NULL); + g_signal_connect (h->internal_session, "on-new-sender-ssrc", + G_CALLBACK (set_min_rtcp_interval_on_new_sender_ssrc), NULL); + + /* Receive a RTP buffer from the wire */ + fail_unless_equals_int (GST_FLOW_OK, + session_harness_recv_rtp (h, generate_test_buffer (0, 0x12345678))); + + /* Wait for first regular RTCP to be sent */ + session_harness_produce_rtcp (h, 1); + buf = session_harness_pull_rtcp (h); + fail_unless (gst_rtcp_buffer_validate (buf)); + gst_buffer_unref (buf); + + session_harness_free (h); +} + +GST_END_TEST; + +G_GNUC_NO_INSTRUMENT static void +_count_warns_log_func (G_GNUC_UNUSED GstDebugCategory * category, + GstDebugLevel level, + G_GNUC_UNUSED const gchar * file, + G_GNUC_UNUSED const gchar * function, + G_GNUC_UNUSED gint line, + G_GNUC_UNUSED GObject * object, + G_GNUC_UNUSED GstDebugMessage * message, gpointer user_data) +{ + if (level == GST_LEVEL_WARNING) { + guint *logged_warnings = user_data; + (*logged_warnings)++; + } +} + +GST_START_TEST (test_twcc_sent_packets_wrap) +{ + SessionHarness *h; + gint i; + guint logged_warnings = 0; + GstDebugLevel level = gst_debug_get_default_threshold (); + + h = session_harness_new (); + session_harness_add_twcc_caps_for_pt (h, TEST_BUF_PT); + + gst_debug_set_default_threshold (GST_LEVEL_WARNING); + gst_debug_add_log_function (_count_warns_log_func, &logged_warnings, NULL); + + for (i = 0; i < 65536; i++) { + GstBuffer *buf; + GstTxFeedbackMeta *meta; + GstClockTime ts; + gboolean marker; + + marker = (i == 65536 - 1); + buf = generate_twcc_send_buffer (i, marker); + ts = i * TEST_BUF_DURATION; + + fail_unless_equals_int64 (session_harness_send_rtp (h, buf), GST_FLOW_OK); + + buf = session_harness_pull_send_rtp (h); + meta = gst_buffer_get_tx_feedback_meta (buf); + fail_unless (meta); + + gst_tx_feedback_meta_set_tx_time (meta, ts); + gst_buffer_unref (buf); + } + + /* demand no warnings have been logged, means we have should have set all + sent packets "send" timestamps */ + fail_unless_equals_uint64 (logged_warnings, 0); + + session_harness_free (h); + + gst_debug_set_default_threshold (level); +} + +GST_END_TEST; + +GST_START_TEST (test_send_rtcp_instantly) +{ + SessionHarness *h = session_harness_new (); + gboolean ret; + const GstClockTime now = 123456789; + + /* advance the clock to "now" */ + gst_test_clock_set_time (h->testclock, now); + + /* verify the RTCP thread has not started */ + fail_unless_equals_int (0, gst_test_clock_peek_id_count (h->testclock)); + /* and that no RTCP has been pushed */ + fail_unless_equals_int (0, gst_harness_buffers_in_queue (h->rtcp_h)); + + /* then ask explicitly to send RTCP with 0 timeout (now!) */ + g_signal_emit_by_name (h->internal_session, "send-rtcp-full", 0, &ret); + /* this is TRUE due to ? */ + fail_unless (ret == TRUE); + + /* "crank" and verify RTCP now was sent */ + session_harness_crank_clock (h); + gst_buffer_unref (session_harness_pull_rtcp (h)); + + /* and check the time is "now" */ + fail_unless_equals_int64 (now, gst_clock_get_time (GST_CLOCK (h->testclock))); + + session_harness_free (h); +} + +GST_END_TEST; + +GST_START_TEST (test_send_bye_signal) +{ + SessionHarness *h = session_harness_new (); + GstBuffer *buf; + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + GstRTCPPacket rtcp_packet; + guint32 ssrc; + + /* push a buffer to establish an internal source */ + fail_unless_equals_int (GST_FLOW_OK, + session_harness_send_rtp (h, generate_test_buffer (0, 0xDEADBEEF))); + + /* emit the signal to signal bye on all sources */ + g_signal_emit_by_name (h->session, "send-bye"); + + session_harness_produce_rtcp (h, 1); + buf = session_harness_pull_rtcp (h); + + fail_unless (gst_rtcp_buffer_validate (buf)); + gst_rtcp_buffer_map (buf, GST_MAP_READ, &rtcp); + /* our RTCP buffer has 3 packets */ + fail_unless_equals_int (3, gst_rtcp_buffer_get_packet_count (&rtcp)); + + /* first a Sender Report */ + fail_unless (gst_rtcp_buffer_get_first_packet (&rtcp, &rtcp_packet)); + fail_unless_equals_int (GST_RTCP_TYPE_SR, + gst_rtcp_packet_get_type (&rtcp_packet)); + gst_rtcp_packet_sr_get_sender_info (&rtcp_packet, &ssrc, NULL, NULL, + NULL, NULL); + fail_unless_equals_int (0xDEADBEEF, ssrc); + fail_unless (gst_rtcp_packet_move_to_next (&rtcp_packet)); + + /* then a SDES */ + fail_unless_equals_int (GST_RTCP_TYPE_SDES, + gst_rtcp_packet_get_type (&rtcp_packet)); + fail_unless_equals_int (0xDEADBEEF, + gst_rtcp_packet_sdes_get_ssrc (&rtcp_packet)); + fail_unless (gst_rtcp_packet_move_to_next (&rtcp_packet)); + + /* and finally the BYE we asked for */ + fail_unless_equals_int (GST_RTCP_TYPE_BYE, + gst_rtcp_packet_get_type (&rtcp_packet)); + + gst_rtcp_buffer_unmap (&rtcp); + gst_buffer_unref (buf); + session_harness_free (h); +} + +GST_END_TEST; + +GST_START_TEST (test_stats_rtcp_with_multiple_rb) +{ + SessionHarness *h = session_harness_new (); + guint j, k; + GstFlowReturn res; + GstRTCPPacket packet; + gint internal_stats_entries; + GstBuffer *buf = NULL; + GstStructure *stats = NULL; + GValueArray *source_stats = NULL; + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + + /* Push RTP from our send SSRCs */ + for (j = 0; j < 5; j++) { /* packets per ssrc */ + for (k = 0; k < 2; k++) { /* number of ssrcs */ + buf = generate_test_buffer (j, 10000 + k); + res = session_harness_send_rtp (h, buf); + fail_unless_equals_int (GST_FLOW_OK, res); + } + } + + /* Push RTCP RR with 2 RBs corresponding to our send SSRCs */ + buf = gst_rtcp_buffer_new (1000); + fail_unless (gst_rtcp_buffer_map (buf, GST_MAP_READWRITE, &rtcp)); + fail_unless (gst_rtcp_buffer_add_packet (&rtcp, GST_RTCP_TYPE_RR, &packet)); + gst_rtcp_packet_rr_set_ssrc (&packet, 20000); + for (k = 0; k < 2; k++) { + guint32 ssrc = 10000 + k; + guint32 jitter = (ssrc % 2) + 10; + gst_rtcp_packet_add_rb (&packet, 10000 + k, /* ssrc */ + 0, /* fractionlost */ + 0, /* packetslost */ + 4, /* exthighestseq */ + jitter, /* jitter */ + 0, /* lsr */ + 0); /* dlsr */ + } + gst_rtcp_buffer_unmap (&rtcp); + fail_unless_equals_int (GST_FLOW_OK, session_harness_recv_rtcp (h, buf)); + + /* Check that the stats reflect the data we received in RTCP */ + internal_stats_entries = 0; + g_object_get (h->session, "stats", &stats, NULL); + fail_unless (stats != NULL); + source_stats = + g_value_get_boxed (gst_structure_get_value (stats, "source-stats")); + fail_unless (source_stats != NULL); + for (j = 0; j < source_stats->n_values; j++) { + guint32 ssrc; + gboolean internal; + GstStructure *s = + g_value_get_boxed (g_value_array_get_nth (source_stats, j)); + fail_unless (gst_structure_get (s, "internal", G_TYPE_BOOLEAN, &internal, + "ssrc", G_TYPE_UINT, &ssrc, NULL)); + if (internal) { + gboolean have_rb; + guint rb_fractionlost; + gint rb_packetslost; + guint rb_exthighestseq; + guint rb_jitter; + guint rb_lsr; + guint rb_dlsr; + + fail_unless_equals_int (ssrc, 10000 + internal_stats_entries); + + fail_unless (gst_structure_get (s, + "have-rb", G_TYPE_BOOLEAN, &have_rb, + "rb-fractionlost", G_TYPE_UINT, &rb_fractionlost, + "rb-packetslost", G_TYPE_INT, &rb_packetslost, + "rb-exthighestseq", G_TYPE_UINT, &rb_exthighestseq, + "rb-jitter", G_TYPE_UINT, &rb_jitter, + "rb-lsr", G_TYPE_UINT, &rb_lsr, + "rb-dlsr", G_TYPE_UINT, &rb_dlsr, NULL)); + fail_unless (have_rb); + fail_unless_equals_int (rb_fractionlost, 0); + fail_unless_equals_int (rb_packetslost, 0); + fail_unless_equals_int (rb_exthighestseq, 4); + fail_unless_equals_int (rb_jitter, (ssrc % 2) + 10); + fail_unless_equals_int (rb_lsr, 0); + fail_unless_equals_int (rb_dlsr, 0); + internal_stats_entries++; + } else { + fail_unless_equals_int (ssrc, 20000); + } + } + fail_unless_equals_int (internal_stats_entries, 2); + + gst_structure_free (stats); + session_harness_free (h); +} + +GST_END_TEST; + +static void +count_report_stats (G_GNUC_UNUSED GObject * object, + G_GNUC_UNUSED GParamSpec * spec, gint * counter) +{ + *counter += 1; +} + +static void +on_sending_rtcp_add_new_if_empty (G_GNUC_UNUSED GObject * rtpsession, + GstBuffer * rtcp_buffer, G_GNUC_UNUSED gboolean is_early, + G_GNUC_UNUSED gpointer user_data) +{ + if (gst_buffer_get_size (rtcp_buffer) == 0) { + GstRTCPPacket packet; + GstRTCPBuffer rtcp = GST_RTCP_BUFFER_INIT; + + gst_rtcp_buffer_map (rtcp_buffer, GST_MAP_READWRITE, &rtcp); + + gst_rtcp_buffer_add_packet (&rtcp, GST_RTCP_TYPE_APP, &packet); + gst_rtcp_packet_app_set_subtype (&packet, 1); + gst_rtcp_packet_app_set_ssrc (&packet, 0x12345678); + gst_rtcp_packet_app_set_name (&packet, "foo"); + + gst_rtcp_buffer_unmap (&rtcp); + } +} + +GST_START_TEST (test_report_stats_only_on_regular_rtcp) +{ + SessionHarness *h = session_harness_new (); + gint stats_callback_count = 0; + gint i; + + g_object_set (h->internal_session, "probation", 1, "rtcp-reduced-size", TRUE, + "stats-notify-min-interval", 3000, NULL); + g_signal_connect (h->session, "notify::stats", + G_CALLBACK (count_report_stats), &stats_callback_count); + + /* Not allowed to send empty packets, so need to add feedback */ + g_signal_connect (h->internal_session, "on-sending-rtcp", + G_CALLBACK (on_sending_rtcp_add_new_if_empty), NULL); + + fail_unless_equals_int (GST_FLOW_OK, + session_harness_recv_rtp (h, generate_test_buffer (0, 0x12345678))); + + session_harness_produce_rtcp (h, 1); + gst_buffer_unref (session_harness_pull_rtcp (h)); + fail_unless_equals_int (stats_callback_count, 1); + + /* send 10 rtcp-packets that should *not* generate stats */ + for (i = 0; i < 10; i++) { + gboolean ret; + g_signal_emit_by_name (h->internal_session, "send-rtcp-full", 0, &ret); + session_harness_advance_and_crank (h, 10 * GST_MSECOND); + } + + /* verify we have generated less than 3 stats for all these packets */ + fail_unless (stats_callback_count < 3); + + session_harness_free (h); +} + +GST_END_TEST; + +static void +copy_stats (GObject * object, G_GNUC_UNUSED GParamSpec * spec, + GstStructure ** stats) +{ + g_object_get (object, "stats", stats, NULL); +} + +static GstCaps * +rtpsession_request_pt_map (G_GNUC_UNUSED GstElement * element, + G_GNUC_UNUSED guint pt, GstCaps * caps) +{ + return gst_caps_copy (caps); +} + +GST_START_TEST (test_stats_transmission_duration) +{ + GstTestClock *testclock = GST_TEST_CLOCK (gst_test_clock_new ()); + GstElement *rtpsession; + GstHarness *h, *h_rtp; + GstStructure *stats = NULL; + GValueArray *source_stats; + gboolean stats_verified = FALSE; + GstCaps *caps = generate_caps (TEST_BUF_PT); + guint i; + + /* use testclock as the systemclock to capture the rtcp thread waits */ + gst_system_clock_set_default (GST_CLOCK (testclock)); + + h = gst_harness_new_with_padnames ("rtpsession", + "recv_rtcp_sink", "send_rtcp_src"); + h_rtp = gst_harness_new_with_element (h->element, + "recv_rtp_sink", "recv_rtp_src"); + + g_signal_connect (h->element, "notify::stats", G_CALLBACK (copy_stats), + &stats); + g_signal_connect (h->element, "request-pt-map", + G_CALLBACK (rtpsession_request_pt_map), caps); + + /* Set probation=1 so that first packet is pushed through immediately. Makes + * test simpler. */ + g_object_get (h->element, "internal-session", &rtpsession, NULL); + g_object_set (rtpsession, "probation", 1, NULL); + + gst_harness_set_src_caps_str (h_rtp, "application/x-rtp"); + + /* first frame has transmission duration of 20 ms */ + gst_test_clock_set_time (testclock, 0 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (0 * GST_MSECOND, + 0, 0 * TEST_BUF_CLOCK_RATE / 10)); + + gst_test_clock_set_time (testclock, 10 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (10 * GST_MSECOND, + 1, 0 * TEST_BUF_CLOCK_RATE / 10)); + + gst_test_clock_set_time (testclock, 20 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (20 * GST_MSECOND, + 2, 0 * TEST_BUF_CLOCK_RATE / 10)); + + /* second frame has transmission duration of 0 ms */ + gst_test_clock_set_time (testclock, 100 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (100 * GST_MSECOND, + 3, 1 * TEST_BUF_CLOCK_RATE / 10)); + + /* need third frame to register that second frame is finished */ + gst_test_clock_set_time (testclock, 200 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (200 * GST_MSECOND, + 4, 2 * TEST_BUF_CLOCK_RATE / 10)); + + /* crank to get the stats */ + gst_test_clock_crank (testclock); + while (stats == NULL) + g_thread_yield (); + fail_unless (stats != NULL); + + source_stats = + g_value_get_boxed (gst_structure_get_value (stats, "source-stats")); + fail_unless (source_stats); + + for (i = 0; i < source_stats->n_values; i++) { + GstStructure *s = + g_value_get_boxed (g_value_array_get_nth (source_stats, i)); + gboolean internal; + gst_structure_get (s, "internal", G_TYPE_BOOLEAN, &internal, NULL); + if (!internal) { + GstClockTime avg_tdur, max_tdur; + gst_structure_get (s, + "avg-frame-transmission-duration", G_TYPE_UINT64, &avg_tdur, + "max-frame-transmission-duration", G_TYPE_UINT64, &max_tdur, NULL); + fail_unless_equals_int (max_tdur, 20 * GST_MSECOND); + fail_unless_equals_int (avg_tdur, (0 + 1023 * 20 * GST_MSECOND) / 1024); + stats_verified = TRUE; + break; + } + } + fail_unless (stats_verified); + + gst_structure_free (stats); + + gst_caps_unref (caps); + gst_object_unref (testclock); + gst_object_unref (rtpsession); + gst_harness_teardown (h_rtp); + gst_harness_teardown (h); + + /* Reset to default system clock */ + gst_system_clock_set_default (NULL); +} + +GST_END_TEST; + +GST_START_TEST (test_stats_transmission_duration_reordering) +{ + GstTestClock *testclock = GST_TEST_CLOCK (gst_test_clock_new ()); + GstElement *rtpsession; + GstHarness *h, *h_rtp; + GstStructure *stats = NULL; + GValueArray *source_stats; + gboolean stats_verified = FALSE; + GstCaps *caps = generate_caps (TEST_BUF_PT); + guint i; + + /* use testclock as the systemclock to capture the rtcp thread waits */ + gst_system_clock_set_default (GST_CLOCK (testclock)); + + h = gst_harness_new_with_padnames ("rtpsession", + "recv_rtcp_sink", "send_rtcp_src"); + h_rtp = gst_harness_new_with_element (h->element, + "recv_rtp_sink", "recv_rtp_src"); + + g_signal_connect (h->element, "notify::stats", G_CALLBACK (copy_stats), + &stats); + g_signal_connect (h->element, "request-pt-map", + G_CALLBACK (rtpsession_request_pt_map), caps); + + /* Set probation=1 so that first packet is pushed through immediately. Makes + * test simpler. */ + g_object_get (h->element, "internal-session", &rtpsession, NULL); + g_object_set (rtpsession, "probation", 1, NULL); + + gst_harness_set_src_caps_str (h_rtp, "application/x-rtp"); + + /* first frame has transmission duration of 20 ms */ + gst_test_clock_set_time (testclock, 0 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (0 * GST_MSECOND, + 0, 0 * TEST_BUF_CLOCK_RATE / 10)); + + gst_test_clock_set_time (testclock, 50 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (50 * GST_MSECOND, + 1, 0 * TEST_BUF_CLOCK_RATE / 10)); + + /* second frame comes before last packet of previous frame */ + gst_test_clock_set_time (testclock, 100 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (100 * GST_MSECOND, + 3, 1 * TEST_BUF_CLOCK_RATE / 10)); + + /* last packet of first frame arrives */ + gst_test_clock_set_time (testclock, 110 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (110 * GST_MSECOND, + 2, 0 * TEST_BUF_CLOCK_RATE / 10)); + + /* need third frame to register that second frame is finished */ + gst_test_clock_set_time (testclock, 200 * GST_MSECOND); + gst_harness_push (h_rtp, generate_test_buffer_timed (200 * GST_MSECOND, + 4, 2 * TEST_BUF_CLOCK_RATE / 10)); + + /* crank to get the stats */ + gst_test_clock_crank (testclock); + while (stats == NULL) + g_thread_yield (); + fail_unless (stats != NULL); + + source_stats = + g_value_get_boxed (gst_structure_get_value (stats, "source-stats")); + fail_unless (source_stats); + + for (i = 0; i < source_stats->n_values; i++) { + GstStructure *s = + g_value_get_boxed (g_value_array_get_nth (source_stats, i)); + gboolean internal; + gst_structure_get (s, "internal", G_TYPE_BOOLEAN, &internal, NULL); + if (!internal) { + GstClockTime avg_tdur, max_tdur; + gst_structure_get (s, + "avg-frame-transmission-duration", G_TYPE_UINT64, &avg_tdur, + "max-frame-transmission-duration", G_TYPE_UINT64, &max_tdur, NULL); + /* the reordered packet will be ignored by stats becuase of + * simplicity */ + fail_unless_equals_int (max_tdur, 50 * GST_MSECOND); + fail_unless_equals_int (avg_tdur, (0 + 1023 * 50 * GST_MSECOND) / 1024); + stats_verified = TRUE; + break; + } + } + fail_unless (stats_verified); + + gst_structure_free (stats); + + gst_caps_unref (caps); + gst_object_unref (testclock); + gst_object_unref (rtpsession); + gst_harness_teardown (h_rtp); + gst_harness_teardown (h); + + /* Reset to default system clock */ + gst_system_clock_set_default (NULL); +} + +GST_END_TEST; static Suite * rtpsession_suite (void) @@ -4323,6 +5867,7 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_ssrc_collision_when_sending_loopback); tcase_add_test (tc_chain, test_ssrc_collision_when_receiving); tcase_add_test (tc_chain, test_ssrc_collision_third_party); + tcase_add_test (tc_chain, test_ssrc_collision_third_party_disable); tcase_add_test (tc_chain, test_ssrc_collision_third_party_favor_new); tcase_add_test (tc_chain, test_ssrc_collision_never_send_on_non_internal_source); @@ -4334,6 +5879,7 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_request_nack_surplus); tcase_add_test (tc_chain, test_request_nack_packing); tcase_add_test (tc_chain, test_illegal_rtcp_fb_packet); + tcase_add_test (tc_chain, test_illegal_rtcp_type_packet); tcase_add_test (tc_chain, test_feedback_rtcp_race); tcase_add_test (tc_chain, test_receive_regular_pli); tcase_add_test (tc_chain, test_receive_pli_no_sender_ssrc); @@ -4347,6 +5893,7 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_clear_pt_map_stress); tcase_add_test (tc_chain, test_packet_rate); tcase_add_test (tc_chain, test_stepped_packet_rate); + tcase_add_test (tc_chain, test_creating_srrr); /* twcc */ tcase_add_loop_test (tc_chain, test_twcc_header_and_run_length, @@ -4367,12 +5914,18 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_twcc_huge_seqnum_gap); tcase_add_test (tc_chain, test_twcc_double_packets); tcase_add_test (tc_chain, test_twcc_duplicate_seqnums); + tcase_add_test (tc_chain, test_twcc_duplicate_previous_seqnums); + tcase_add_test (tc_chain, test_twcc_missing_packet_duplicates_last); tcase_add_test (tc_chain, test_twcc_multiple_markers); tcase_add_test (tc_chain, test_twcc_no_marker_and_gaps); tcase_add_test (tc_chain, test_twcc_bad_rtcp); tcase_add_test (tc_chain, test_twcc_delta_ts_rounding); tcase_add_test (tc_chain, test_twcc_double_gap); tcase_add_test (tc_chain, test_twcc_recv_packets_reordered); + tcase_add_test (tc_chain, test_twcc_recv_packets_reordered_and_lost); + tcase_add_test (tc_chain, + test_twcc_recv_packets_reordered_within_report_interval); + tcase_add_test (tc_chain, test_twcc_reordering_send_recv); tcase_add_test (tc_chain, test_twcc_recv_late_packet_fb_pkt_count_wrap); tcase_add_test (tc_chain, test_twcc_recv_rtcp_reordered); tcase_add_test (tc_chain, test_twcc_no_exthdr_in_buffer); @@ -4380,9 +5933,23 @@ rtpsession_suite (void) tcase_add_test (tc_chain, test_twcc_multiple_payloads_below_window); tcase_add_loop_test (tc_chain, test_twcc_feedback_interval, 0, G_N_ELEMENTS (test_twcc_feedback_interval_ctx)); + tcase_add_test (tc_chain, test_twcc_feedback_interval_new_internal_source); tcase_add_test (tc_chain, test_twcc_feedback_count_wrap); tcase_add_test (tc_chain, test_twcc_feedback_old_seqnum); - + tcase_add_test (tc_chain, test_twcc_stats_rtx_recover_lost); + tcase_add_test (tc_chain, test_twcc_stats_no_rtx_no_recover); + tcase_add_test (tc_chain, test_twcc_feedback_max_sent_packets); + tcase_add_test (tc_chain, test_twcc_non_twcc_pkts_does_not_mark_loss); + tcase_add_test (tc_chain, test_twcc_non_twcc_pt_no_twcc_seqnum); + tcase_add_test (tc_chain, test_twcc_overwrites_exthdr_seqnum_if_present); + tcase_add_test (tc_chain, test_twcc_sent_packets_wrap); + + tcase_add_test (tc_chain, test_send_rtcp_instantly); + tcase_add_test (tc_chain, test_send_bye_signal); + tcase_skip_broken_test (tc_chain, test_stats_rtcp_with_multiple_rb); + tcase_add_test (tc_chain, test_report_stats_only_on_regular_rtcp); + tcase_add_test (tc_chain, test_stats_transmission_duration); + tcase_add_test (tc_chain, test_stats_transmission_duration_reordering); return s; } diff --git a/subprojects/gst-plugins-good/tests/check/meson.build b/subprojects/gst-plugins-good/tests/check/meson.build index 3a4b5c99a67..4b29b16bb31 100644 --- a/subprojects/gst-plugins-good/tests/check/meson.build +++ b/subprojects/gst-plugins-good/tests/check/meson.build @@ -87,6 +87,7 @@ good_tests = [ if not get_option('rtp').disabled() and not get_option('rtpmanager').disabled() good_tests += [ [ 'elements/rtphdrext-colorspace' ], + [ 'elements/rtphdrext-roi' ], [ 'elements/rtph261', ], [ 'elements/rtph263' ], [ 'elements/rtph264' ], @@ -103,7 +104,6 @@ if not get_option('rtp').disabled() and not get_option('rtpmanager').disabled() [ 'elements/rtphdrextsdes', false, [gstrtp_dep, gstsdp_dep] ], [ 'elements/rtpjitterbuffer' ], [ 'elements/rtpjpeg' ], - [ 'elements/rtptimerqueue', false, [gstrtp_dep], ['../../gst/rtpmanager/rtptimerqueue.c']], diff --git a/subprojects/gstreamer/gst/gst.c b/subprojects/gstreamer/gst/gst.c index 3f9096197ba..a99a678f6b3 100644 --- a/subprojects/gstreamer/gst/gst.c +++ b/subprojects/gstreamer/gst/gst.c @@ -1088,7 +1088,9 @@ gst_deinit (void) } if (gst_deinitialized) { /* tell the user how naughty they've been */ - g_error ("GStreamer should not be deinitialized a second time."); + g_message ("GStreamer should not be deinitialized a second time."); + g_mutex_unlock (&init_lock); + return; } GST_INFO ("deinitializing GStreamer"); diff --git a/subprojects/gstreamer/gst/gst_private.h b/subprojects/gstreamer/gst/gst_private.h index 9be91ef96e4..0c7a01dc922 100644 --- a/subprojects/gstreamer/gst/gst_private.h +++ b/subprojects/gstreamer/gst/gst_private.h @@ -512,7 +512,7 @@ struct _GstDynamicTypeFactoryClass { struct _GstClockEntryImpl { GstClockEntry entry; - GWeakRef *clock; + GstClock *clock; GDestroyNotify destroy_entry; gpointer padding[21]; /* padding for allowing e.g. systemclock * to add data in lieu of overridable diff --git a/subprojects/gstreamer/gst/gstbin.c b/subprojects/gstreamer/gst/gstbin.c index 8c66e35c52a..64afef944c7 100644 --- a/subprojects/gstreamer/gst/gstbin.c +++ b/subprojects/gstreamer/gst/gstbin.c @@ -1183,8 +1183,9 @@ gst_bin_add_func (GstBin * bin, GstElement * element) * we can safely take the lock here. This check is probably bogus because * you can safely change the element name after this check and before setting * the object parent. The window is very small though... */ - if (G_UNLIKELY (!gst_object_check_uniqueness (bin->children, elem_name))) - goto duplicate_name; + if (!GST_OBJECT_FLAG_IS_SET (element, GST_ELEMENT_FLAG_NO_UNIQUE_CHECK)) + if (G_UNLIKELY (!gst_object_check_uniqueness (bin->children, elem_name))) + goto duplicate_name; /* set the element's parent and add the element to the bin's list of children */ if (G_UNLIKELY (!gst_object_set_parent (GST_OBJECT_CAST (element), diff --git a/subprojects/gstreamer/gst/gstclock.c b/subprojects/gstreamer/gst/gstclock.c index ab8a4b00255..df7bba3b315 100644 --- a/subprojects/gstreamer/gst/gstclock.c +++ b/subprojects/gstreamer/gst/gstclock.c @@ -134,6 +134,9 @@ enum struct _GstClockPrivate { + /* Not locked, but only atomic use */ + gint weak_refcount; + GMutex slave_lock; /* order: SLAVE_LOCK, OBJECT_LOCK */ GCond sync_cond; @@ -165,13 +168,11 @@ struct _GstClockPrivate gint post_count; gboolean synced; - - GWeakRef *clock_weakref; }; typedef struct _GstClockEntryImpl GstClockEntryImpl; -#define GST_CLOCK_ENTRY_CLOCK_WEAK_REF(entry) (((GstClockEntryImpl *)(entry))->clock) +#define GST_CLOCK_ENTRY_CLOCK_GET(entry) (((GstClockEntryImpl *)(entry))->clock) /* seqlocks */ #define read_seqbegin(clock) \ @@ -262,8 +263,8 @@ gst_clock_entry_new (GstClock * clock, GstClockTime time, entry->_clock = clock; #endif #endif - GST_CLOCK_ENTRY_CLOCK_WEAK_REF (entry) = - g_atomic_rc_box_acquire (clock->priv->clock_weakref); + GST_CLOCK_ENTRY_CLOCK_GET (entry) = clock; + g_atomic_int_inc (&(GST_CLOCK_ENTRY_CLOCK_GET (entry)->priv->weak_refcount)); entry->type = type; entry->time = time; entry->interval = interval; @@ -377,9 +378,10 @@ _gst_clock_id_free (GstClockID id) if (entry_impl->destroy_entry) entry_impl->destroy_entry (entry_impl); - g_atomic_rc_box_release_full (GST_CLOCK_ENTRY_CLOCK_WEAK_REF (entry), - (GDestroyNotify) g_weak_ref_clear); - + gboolean weak_leak = + g_atomic_int_dec_and_test (&(GST_CLOCK_ENTRY_CLOCK_GET (entry)-> + priv->weak_refcount)); + g_assert (!weak_leak); /* FIXME: add tracer hook for struct allocations such as clock entries */ g_free (id); @@ -535,7 +537,7 @@ gst_clock_id_wait (GstClockID id, GstClockTimeDiff * jitter) entry = (GstClockEntry *) id; requested = GST_CLOCK_ENTRY_TIME (entry); - clock = g_weak_ref_get (GST_CLOCK_ENTRY_CLOCK_WEAK_REF (entry)); + clock = GST_CLOCK_ENTRY_CLOCK_GET (entry); if (G_UNLIKELY (clock == NULL)) goto invalid_entry; @@ -560,7 +562,6 @@ gst_clock_id_wait (GstClockID id, GstClockTimeDiff * jitter) if (entry->type == GST_CLOCK_ENTRY_PERIODIC) entry->time = requested + entry->interval; - gst_object_unref (clock); return res; /* ERRORS */ @@ -568,13 +569,11 @@ gst_clock_id_wait (GstClockID id, GstClockTimeDiff * jitter) { GST_CAT_DEBUG_OBJECT (GST_CAT_CLOCK, clock, "invalid time requested, returning _BADTIME"); - gst_object_unref (clock); return GST_CLOCK_BADTIME; } not_supported: { GST_CAT_DEBUG_OBJECT (GST_CAT_CLOCK, clock, "clock wait is not supported"); - gst_object_unref (clock); return GST_CLOCK_UNSUPPORTED; } invalid_entry: @@ -617,7 +616,7 @@ gst_clock_id_wait_async (GstClockID id, entry = (GstClockEntry *) id; requested = GST_CLOCK_ENTRY_TIME (entry); - clock = g_weak_ref_get (GST_CLOCK_ENTRY_CLOCK_WEAK_REF (entry)); + clock = GST_CLOCK_ENTRY_CLOCK_GET (entry); if (G_UNLIKELY (clock == NULL)) goto invalid_entry; @@ -636,7 +635,6 @@ gst_clock_id_wait_async (GstClockID id, res = cclass->wait_async (clock, entry); - gst_object_unref (clock); return res; /* ERRORS */ @@ -645,13 +643,11 @@ gst_clock_id_wait_async (GstClockID id, (func) (clock, GST_CLOCK_TIME_NONE, id, user_data); GST_CAT_DEBUG_OBJECT (GST_CAT_CLOCK, clock, "invalid time requested, returning _BADTIME"); - gst_object_unref (clock); return GST_CLOCK_BADTIME; } not_supported: { GST_CAT_DEBUG_OBJECT (GST_CAT_CLOCK, clock, "clock wait is not supported"); - gst_object_unref (clock); return GST_CLOCK_UNSUPPORTED; } invalid_entry: @@ -680,7 +676,7 @@ gst_clock_id_unschedule (GstClockID id) g_return_if_fail (id != NULL); entry = (GstClockEntry *) id; - clock = g_weak_ref_get (GST_CLOCK_ENTRY_CLOCK_WEAK_REF (entry)); + clock = GST_CLOCK_ENTRY_CLOCK_GET (entry); if (G_UNLIKELY (clock == NULL)) goto invalid_entry; @@ -689,7 +685,6 @@ gst_clock_id_unschedule (GstClockID id) if (G_LIKELY (cclass->unschedule)) cclass->unschedule (clock, entry); - gst_object_unref (clock); return; invalid_entry: @@ -757,6 +752,8 @@ gst_clock_init (GstClock * clock) clock->priv = priv = gst_clock_get_instance_private (clock); + priv->weak_refcount = 1; /* Set it to 1 so we can assert on dec and test */ + priv->last_time = 0; priv->internal_calibration = 0; @@ -773,17 +770,6 @@ gst_clock_init (GstClock * clock) priv->timeout = DEFAULT_TIMEOUT; priv->times = g_new0 (GstClockTime, 4 * priv->window_size); priv->times_temp = priv->times + 2 * priv->window_size; - /* - * An atomically ref-counted wrapper around a GWeakRef for this GstClock, - * created by the clock and shared with all its clock entries. - * - * This exists because g_weak_ref_ operations are quite expensive and operate - * with a global GRWLock. _get takes a reader lock, _init and _clear take - * a writer lock. We want to avoid having to instantiate a new GWeakRef for - * every clock entry. - */ - priv->clock_weakref = g_atomic_rc_box_new (GWeakRef); - g_weak_ref_init (priv->clock_weakref, clock); } static void @@ -793,6 +779,11 @@ gst_clock_dispose (GObject * object) GstClock **master_p; GST_OBJECT_LOCK (clock); + + /* Ensure we have no dangeling weak pointers */ + int num_weak_refs = g_atomic_int_get (&(clock->priv->weak_refcount)); + g_assert (num_weak_refs == 1); + master_p = &clock->priv->master; gst_object_replace ((GstObject **) master_p, NULL); GST_OBJECT_UNLOCK (clock); @@ -816,8 +807,6 @@ gst_clock_finalize (GObject * object) clock->priv->times_temp = NULL; GST_CLOCK_SLAVE_UNLOCK (clock); - g_atomic_rc_box_release_full (clock->priv->clock_weakref, - (GDestroyNotify) g_weak_ref_clear); g_mutex_clear (&clock->priv->slave_lock); g_cond_clear (&clock->priv->sync_cond); @@ -1391,7 +1380,7 @@ gst_clock_id_get_clock (GstClockID id) g_return_val_if_fail (id != NULL, NULL); entry = (GstClockEntry *) id; - return g_weak_ref_get (GST_CLOCK_ENTRY_CLOCK_WEAK_REF (entry)); + return gst_object_ref (GST_CLOCK_ENTRY_CLOCK_GET (entry)); } /** @@ -1418,13 +1407,10 @@ gst_clock_id_uses_clock (GstClockID id, GstClock * clock) g_return_val_if_fail (clock != NULL, FALSE); entry = (GstClockEntry *) id; - entry_clock = g_weak_ref_get (GST_CLOCK_ENTRY_CLOCK_WEAK_REF (entry)); + entry_clock = GST_CLOCK_ENTRY_CLOCK_GET (entry); if (entry_clock == clock) ret = TRUE; - if (G_LIKELY (entry_clock != NULL)) - gst_object_unref (entry_clock); - return ret; } diff --git a/subprojects/gstreamer/gst/gstdeviceprovider.c b/subprojects/gstreamer/gst/gstdeviceprovider.c index f073ede51b7..1699047e929 100644 --- a/subprojects/gstreamer/gst/gstdeviceprovider.c +++ b/subprojects/gstreamer/gst/gstdeviceprovider.c @@ -180,8 +180,13 @@ gst_device_provider_dispose (GObject * object) gst_object_replace ((GstObject **) & provider->priv->bus, NULL); GST_OBJECT_LOCK (provider); + g_list_free_full (provider->devices, (GDestroyNotify) gst_object_unparent); provider->devices = NULL; + + g_list_free_full (provider->priv->hidden_providers, (GDestroyNotify) g_free); + provider->priv->hidden_providers = NULL; + GST_OBJECT_UNLOCK (provider); G_OBJECT_CLASS (gst_device_provider_parent_class)->dispose (object); diff --git a/subprojects/gstreamer/gst/gstelement.c b/subprojects/gstreamer/gst/gstelement.c index 10ce62ba62c..e5919a21376 100644 --- a/subprojects/gstreamer/gst/gstelement.c +++ b/subprojects/gstreamer/gst/gstelement.c @@ -763,8 +763,9 @@ gst_element_add_pad (GstElement * element, GstPad * pad) /* then check to see if there's already a pad by that name here */ GST_OBJECT_LOCK (element); - if (G_UNLIKELY (!gst_object_check_uniqueness (element->pads, pad_name))) - goto name_exists; + if (!GST_OBJECT_FLAG_IS_SET (element, GST_ELEMENT_FLAG_NO_UNIQUE_CHECK)) + if (G_UNLIKELY (!gst_object_check_uniqueness (element->pads, pad_name))) + goto name_exists; /* try to set the pad's parent */ if (G_UNLIKELY (!gst_object_set_parent (GST_OBJECT_CAST (pad), @@ -1268,8 +1269,11 @@ gst_element_request_pad_simple (GstElement * element, const gchar * name) } } - if (!templ_found) + if (!templ_found) { + GST_CAT_WARNING (GST_CAT_PADS, "Could not find template in %" GST_PTR_FORMAT + " to match requested pad name %s", element, name); return NULL; + } pad = _gst_element_request_pad (element, templ, req_name, NULL); diff --git a/subprojects/gstreamer/gst/gstelement.h b/subprojects/gstreamer/gst/gstelement.h index e690bf0e3f0..59e03e8f5d4 100644 --- a/subprojects/gstreamer/gst/gstelement.h +++ b/subprojects/gstreamer/gst/gstelement.h @@ -407,12 +407,13 @@ typedef enum /*< flags=0 >*/ */ typedef enum { - GST_ELEMENT_FLAG_LOCKED_STATE = (GST_OBJECT_FLAG_LAST << 0), - GST_ELEMENT_FLAG_SINK = (GST_OBJECT_FLAG_LAST << 1), - GST_ELEMENT_FLAG_SOURCE = (GST_OBJECT_FLAG_LAST << 2), - GST_ELEMENT_FLAG_PROVIDE_CLOCK = (GST_OBJECT_FLAG_LAST << 3), - GST_ELEMENT_FLAG_REQUIRE_CLOCK = (GST_OBJECT_FLAG_LAST << 4), - GST_ELEMENT_FLAG_INDEXABLE = (GST_OBJECT_FLAG_LAST << 5), + GST_ELEMENT_FLAG_LOCKED_STATE = (GST_OBJECT_FLAG_LAST << 0), + GST_ELEMENT_FLAG_SINK = (GST_OBJECT_FLAG_LAST << 1), + GST_ELEMENT_FLAG_SOURCE = (GST_OBJECT_FLAG_LAST << 2), + GST_ELEMENT_FLAG_PROVIDE_CLOCK = (GST_OBJECT_FLAG_LAST << 3), + GST_ELEMENT_FLAG_REQUIRE_CLOCK = (GST_OBJECT_FLAG_LAST << 4), + GST_ELEMENT_FLAG_INDEXABLE = (GST_OBJECT_FLAG_LAST << 5), + GST_ELEMENT_FLAG_NO_UNIQUE_CHECK = (GST_OBJECT_FLAG_LAST << 6), /* padding */ GST_ELEMENT_FLAG_LAST = (GST_OBJECT_FLAG_LAST << 10) } GstElementFlags; diff --git a/subprojects/gstreamer/gst/gstevent.c b/subprojects/gstreamer/gst/gstevent.c index 5032f1ce9d6..7a2b1640848 100644 --- a/subprojects/gstreamer/gst/gstevent.c +++ b/subprojects/gstreamer/gst/gstevent.c @@ -122,6 +122,7 @@ static GstEventQuarks event_quarks[] = { {GST_EVENT_SEEK, "seek", 0}, {GST_EVENT_NAVIGATION, "navigation", 0}, {GST_EVENT_LATENCY, "latency", 0}, + {GST_EVENT_LATENCY_CHANGED, "latency-changed", 0}, {GST_EVENT_STEP, "step", 0}, {GST_EVENT_RECONFIGURE, "reconfigure", 0}, {GST_EVENT_TOC_SELECT, "toc-select", 0}, @@ -1583,6 +1584,30 @@ gst_event_parse_latency (GstEvent * event, GstClockTime * latency) (event), GST_QUARK (LATENCY))); } +/** + * gst_event_new_latency_changed: + * + * Create a new latency-changed event. The event is sent downstream from + * an element that changes its latency while running, notifying any + * downstream element of this change. + * + * This information is most useful in mixer and sinks, that might send + * an upstream latency-query as a response to figure out the new latency. + * + * Returns: (transfer full): a new #GstEvent + */ +GstEvent * +gst_event_new_latency_changed (void) +{ + GstEvent *event; + + GST_CAT_INFO (GST_CAT_EVENT, "creating latency-changed event"); + + event = gst_event_new_custom (GST_EVENT_LATENCY_CHANGED, NULL); + + return event; +} + /** * gst_event_new_step: * @format: the format of @amount diff --git a/subprojects/gstreamer/gst/gstevent.h b/subprojects/gstreamer/gst/gstevent.h index a58eb40aa82..7b1a1eef708 100644 --- a/subprojects/gstreamer/gst/gstevent.h +++ b/subprojects/gstreamer/gst/gstevent.h @@ -116,6 +116,8 @@ typedef enum { * to upstream elements. * @GST_EVENT_LATENCY: Notification of new latency adjustment. Sinks will use * the latency information to adjust their synchronisation. + * @GST_EVENT_LATENCY_CHANGED: Notification of new latency adjustment upstream. + Sinks can use this to query upstream for the new latency. * @GST_EVENT_STEP: A request for stepping through the media. Sinks will usually * execute the step operation. * @GST_EVENT_RECONFIGURE: A request for upstream renegotiating caps and reconfiguring. @@ -167,6 +169,7 @@ typedef enum { /* non-sticky downstream serialized */ GST_EVENT_SEGMENT_DONE = GST_EVENT_MAKE_TYPE (150, _FLAG(DOWNSTREAM) | _FLAG(SERIALIZED)), GST_EVENT_GAP = GST_EVENT_MAKE_TYPE (160, _FLAG(DOWNSTREAM) | _FLAG(SERIALIZED)), + GST_EVENT_LATENCY_CHANGED = GST_EVENT_MAKE_TYPE (170, _FLAG(DOWNSTREAM) | _FLAG(SERIALIZED)), /* sticky downstream non-serialized */ /* FIXME 2.0: change to value 72 and move after the GST_EVENT_SEGMENT event */ @@ -684,6 +687,10 @@ GstEvent* gst_event_new_latency (GstClockTime latency) G_GNUC_MA GST_API void gst_event_parse_latency (GstEvent *event, GstClockTime *latency); +/* latency-changed event */ +GST_API +GstEvent* gst_event_new_latency_changed (void) G_GNUC_MALLOC; + /* step event */ GST_API diff --git a/subprojects/gstreamer/gst/gstghostpad.c b/subprojects/gstreamer/gst/gstghostpad.c index 99f1cf53e39..6041031b4a4 100644 --- a/subprojects/gstreamer/gst/gstghostpad.c +++ b/subprojects/gstreamer/gst/gstghostpad.c @@ -59,8 +59,12 @@ #define GST_PROXY_PAD_ACQUIRE_INTERNAL(pad, internal, retval) \ internal = \ GST_PAD_CAST (gst_proxy_pad_get_internal (GST_PROXY_PAD_CAST (pad))); \ - if (internal == NULL) \ - return retval; + if (internal == NULL) {\ + if ((GstFlowReturn)retval == GST_FLOW_NOT_LINKED) {\ + GST_WARNING_OBJECT ((pad), "%p:%p returning not-linked", GST_PAD_PARENT(pad), (pad));\ + }\ + return retval;\ + } #define GST_PROXY_PAD_RELEASE_INTERNAL(internal) gst_object_unref (internal); diff --git a/subprojects/gstreamer/gst/gstinfo.c b/subprojects/gstreamer/gst/gstinfo.c index e3872197326..6410b86bc97 100644 --- a/subprojects/gstreamer/gst/gstinfo.c +++ b/subprojects/gstreamer/gst/gstinfo.c @@ -99,6 +99,10 @@ #include #include /* G_VA_COPY */ +#ifdef ENABLE_GST_DEBUG_SYSLOG +# include +#endif + #include "gst_private.h" #include "gstutils.h" #include "gstquark.h" @@ -264,6 +268,10 @@ GstDebugCategory *_priv_GST_CAT_PROTECTION = NULL; #ifndef GST_DISABLE_GST_DEBUG +#ifdef ENABLE_GST_DEBUG_SYSLOG +static gboolean _gst_debug_use_syslog = FALSE; +#endif + /* underscore is to prevent conflict with GST_CAT_DEBUG define */ GST_DEBUG_CATEGORY_STATIC (_GST_CAT_DEBUG); @@ -415,6 +423,14 @@ _priv_gst_debug_init (void) gst_debug_add_log_function (gst_debug_log_default, log_file, NULL); } +#ifdef ENABLE_GST_DEBUG_SYSLOG + env = g_getenv ("GST_DEBUG_SYSLOG"); + if (env != NULL && *env != '\0') { + _gst_debug_use_syslog = TRUE; + openlog (env, LOG_NDELAY | LOG_PID, LOG_LOCAL3); + } +#endif + __gst_printf_pointer_extension_set_func (gst_info_printf_pointer_extension_func); @@ -1665,6 +1681,17 @@ gst_debug_log_default (GstDebugCategory * category, GstDebugLevel level, } FFLUSH_DEBUG (log_file); } + +#ifdef ENABLE_GST_DEBUG_SYSLOG + if (_gst_debug_use_syslog) { +#define PRINT_FMT " "PID_FMT" "PTR_FMT" %s "CAT_FMT" %s\n" + syslog (LOG_INFO|LOG_LOCAL3, "%" GST_TIME_FORMAT PRINT_FMT, GST_TIME_ARGS (elapsed), + pid, g_thread_self (), gst_debug_level_get_name (level), + gst_debug_category_get_name (category), file, line, function, + object_id ? object_id : "", gst_debug_message_get (message)); +#undef PRINT_FMT + } +#endif } /** @@ -2417,15 +2444,8 @@ gst_debug_set_threshold_from_string (const gchar * list, gboolean reset) const gchar *category; if (parse_debug_category (values[0], &category) - && parse_debug_level (values[1], &level)) { + && parse_debug_level (values[1], &level)) gst_debug_set_threshold_for_name (category, level); - - /* bump min-level anyway to allow the category to be registered in the - * future still */ - if (level > _gst_debug_min) { - _gst_debug_min = level; - } - } } g_strfreev (values); @@ -3492,7 +3512,7 @@ gst_debug_print_stack_trace (void) G_LOCK (win_print_mutex); #endif - g_print ("%s\n", trace); + g_printerr ("%s\n", trace); #ifdef G_OS_WIN32 G_UNLOCK (win_print_mutex); diff --git a/subprojects/gstreamer/gst/gstinfo.h b/subprojects/gstreamer/gst/gstinfo.h index 416bd7c25f3..53dccfa541e 100644 --- a/subprojects/gstreamer/gst/gstinfo.h +++ b/subprojects/gstreamer/gst/gstinfo.h @@ -239,6 +239,26 @@ struct _GstDebugCategory { */ #define GST_STR_NULL(str) ((str) ? (str) : "(NULL)") +#if defined (__GNUC__) +#define _GST_OBJECT_PARENT_NAME(obj) \ + ({ \ + GstObject * parent; \ + const gchar * parent_name; \ + if (G_LIKELY ((parent = GST_OBJECT_PARENT (obj)) != NULL)) { \ + /* parent may be disposed, but accessing name will not crash*/ \ + parent_name = GST_OBJECT_NAME (parent); \ + } else { \ + parent_name = "''"; \ + } \ + parent_name; \ + }) +#else +/* Not MT safe since object may be unparented after check */ +#define _GST_OBJECT_PARENT_NAME(obj) \ + (GST_OBJECT_PARENT (obj) != NULL) ? \ + GST_OBJECT_NAME (GST_OBJECT_PARENT (obj)) : "''" +#endif + /* FIXME, not MT safe */ /** * GST_DEBUG_PAD_NAME: @@ -248,10 +268,7 @@ struct _GstDebugCategory { * statements. */ #define GST_DEBUG_PAD_NAME(pad) \ - (pad != NULL) ? \ - ((GST_OBJECT_PARENT(pad) != NULL) ? \ - GST_STR_NULL (GST_OBJECT_NAME (GST_OBJECT_PARENT(pad))) : \ - "''" ) : "''", \ + (pad != NULL) ? GST_STR_NULL (_GST_OBJECT_PARENT_NAME (pad)) : "''", \ (pad != NULL) ? GST_STR_NULL (GST_OBJECT_NAME (pad)) : "''" /** diff --git a/subprojects/gstreamer/gst/gstmacros.h b/subprojects/gstreamer/gst/gstmacros.h index b3313e4e370..3216a94f78d 100644 --- a/subprojects/gstreamer/gst/gstmacros.h +++ b/subprojects/gstreamer/gst/gstmacros.h @@ -52,6 +52,9 @@ G_BEGIN_DECLS # endif #endif +#define GST_CONTAINER_OF(addr, type, member) \ + ((type * const) ((guint8 *) (addr) - G_STRUCT_OFFSET (type, member))) + G_END_DECLS #endif /* __GST_MACROS_H__ */ diff --git a/subprojects/gstreamer/gst/gstpad.c b/subprojects/gstreamer/gst/gstpad.c index a83f25ec861..515c28fcd80 100644 --- a/subprojects/gstreamer/gst/gstpad.c +++ b/subprojects/gstreamer/gst/gstpad.c @@ -150,6 +150,8 @@ struct _GstPadPrivate * by a single thread at a time. Protected by the object lock */ GCond activation_cond; gboolean in_activation; + + gboolean warned_unlinked; }; typedef struct @@ -223,6 +225,20 @@ static GstFlowQuarks flow_quarks[] = { {GST_FLOW_CUSTOM_ERROR, "custom-error", 0} }; +static void +gst_pad_warn_if_unlinked (GstPad * pad, GstFlowReturn res) +{ + if (G_UNLIKELY (GST_PAD_PEER (pad) == NULL && res == GST_FLOW_NOT_LINKED)) { + if (!pad->priv->warned_unlinked) { + pad->priv->warned_unlinked = TRUE; + GST_WARNING_OBJECT (pad, "%p:%p returning not-linked", + GST_PAD_PARENT (pad), pad); + } + } else { + pad->priv->warned_unlinked = FALSE; + } +} + /** * gst_flow_get_name: * @ret: a #GstFlowReturn to get the name of. @@ -417,6 +433,7 @@ gst_pad_init (GstPad * pad) pad->priv->events = g_array_sized_new (FALSE, TRUE, sizeof (PadEvent), 16); pad->priv->events_cookie = 0; pad->priv->last_cookie = -1; + pad->priv->warned_unlinked = FALSE; g_cond_init (&pad->priv->activation_cond); pad->ABI.abi.last_flowret = GST_FLOW_FLUSHING; @@ -2292,20 +2309,23 @@ gst_pad_link_check_compatible_unlocked (GstPad * src, GstPad * sink, /* if we have caps on both pads we can check the intersection. If one * of the caps is %NULL, we return %TRUE. */ if (G_UNLIKELY (srccaps == NULL || sinkcaps == NULL)) { - if (srccaps) - gst_caps_unref (srccaps); - if (sinkcaps) - gst_caps_unref (sinkcaps); goto done; } compatible = gst_caps_can_intersect (srccaps, sinkcaps); - gst_caps_unref (srccaps); - gst_caps_unref (sinkcaps); done: GST_CAT_DEBUG (GST_CAT_CAPS, "caps are %scompatible", (compatible ? "" : "not ")); + if (!compatible) { + GST_CAT_WARNING (GST_CAT_CAPS, "caps are not compatible: %" GST_PTR_FORMAT + " and %" GST_PTR_FORMAT, srccaps, sinkcaps); + } + + if (srccaps) + gst_caps_unref (srccaps); + if (sinkcaps) + gst_caps_unref (sinkcaps); return compatible; } @@ -4857,6 +4877,7 @@ gst_pad_push (GstPad * pad, GstBuffer * buffer) GST_TRACER_PAD_PUSH_PRE (pad, buffer); res = gst_pad_push_data (pad, GST_PAD_PROBE_TYPE_BUFFER | GST_PAD_PROBE_TYPE_PUSH, buffer); + gst_pad_warn_if_unlinked (pad, res); GST_TRACER_PAD_PUSH_POST (pad, res); return res; } @@ -4897,6 +4918,7 @@ gst_pad_push_list (GstPad * pad, GstBufferList * list) GST_TRACER_PAD_PUSH_LIST_PRE (pad, list); res = gst_pad_push_data (pad, GST_PAD_PROBE_TYPE_BUFFER_LIST | GST_PAD_PROBE_TYPE_PUSH, list); + gst_pad_warn_if_unlinked (pad, res); GST_TRACER_PAD_PUSH_LIST_POST (pad, res); return res; } @@ -5293,6 +5315,7 @@ gst_pad_pull_range (GstPad * pad, guint64 offset, guint size, } done: GST_TRACER_PAD_PULL_RANGE_POST (pad, NULL, ret); + gst_pad_warn_if_unlinked (pad, ret); return ret; } diff --git a/subprojects/gstreamer/gst/gstparamspecs.c b/subprojects/gstreamer/gst/gstparamspecs.c index 39c378ce411..f8545adc9eb 100644 --- a/subprojects/gstreamer/gst/gstparamspecs.c +++ b/subprojects/gstreamer/gst/gstparamspecs.c @@ -1,5 +1,6 @@ /* GStreamer - GParamSpecs for some of our types * Copyright (C) 2007 Tim-Philipp Müller + * Copyright (C) 2014 Haakon Sporsheim * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -367,3 +368,147 @@ gst_param_spec_array (const gchar * name, return G_PARAM_SPEC (aspec); } + +/* --- GstParamSpecIntRange --- */ + +static void +_gst_param_int_range_init (GParamSpec * pspec) +{ + GstParamSpecIntRange *irspec = GST_PARAM_SPEC_INT_RANGE (pspec); + + irspec->min_min = G_MININT; + irspec->min_max = G_MININT; + irspec->min_step = G_MININT; + irspec->max_min = G_MAXINT; + irspec->max_max = G_MAXINT; + irspec->max_step = G_MAXINT; + + irspec->def_min = G_MININT; + irspec->def_max = G_MAXINT; + irspec->def_step = 1; +} + +static void +_gst_param_int_range_set_default (GParamSpec * pspec, GValue * value) +{ + GstParamSpecIntRange *irspec = GST_PARAM_SPEC_INT_RANGE (pspec); + + gst_value_set_int_range_step (value, + irspec->def_min, irspec->def_max, irspec->def_step); +} + +static gboolean +_gst_param_int_range_validate (GParamSpec * pspec, GValue * value) +{ + (void)pspec; + + g_return_val_if_fail (GST_VALUE_HOLDS_INT_RANGE (value), TRUE); + g_return_val_if_fail (gst_value_get_int_range_step (value) > 0, TRUE); + + if (gst_value_get_int_range_max (value) < gst_value_get_int_range_min (value)) { + gst_value_set_int_range (value, gst_value_get_int_range_min (value), + gst_value_get_int_range_min (value)); + return TRUE; + } + + return FALSE; +} + +static gint +_gst_param_int_range_values_cmp (GParamSpec * pspec, const GValue * value1, + const GValue * value2) +{ + /* GST_VALUE_LESS_THAN is -1, EQUAL is 0, and GREATER_THAN is 1 */ + return gst_value_compare (value1, value2); +} + +GType +gst_param_spec_int_range_get_type (void) +{ + static GType type; /* 0 */ + + /* register GST_TYPE_PARAM_INT_RANGE */ + if (type == 0) { + static GParamSpecTypeInfo pspec_info = { + sizeof (GstParamSpecIntRange), /* instance_size */ + 0, /* n_preallocs */ + _gst_param_int_range_init, /* instance_init */ + G_TYPE_INVALID, /* value_type */ + NULL, /* finalize */ + _gst_param_int_range_set_default, /* value_set_default */ + _gst_param_int_range_validate, /* value_validate */ + _gst_param_int_range_values_cmp, /* values_cmp */ + }; + pspec_info.value_type = GST_TYPE_INT_RANGE; + type = g_param_type_register_static ("GstParamIntRange", &pspec_info); + } + return type; +} + +/** + * gst_param_spec_int_range: + * @name: canonical name of the property specified + * @nick: nick name for the property specified + * @blurb: description of the property specified + * @min_min: minimum min value + * @min_max: minimum max value + * @min_step: minimum step + * @max_min: maximum min value + * @max_max: maximum max value + * @max_step: maximum step + * @def_min: default min value + * @def_max: default max value + * @def_step: default step + * @flags: flags for the property specified + * + * This function creates an int range GParamSpec for use by objects/elements + * that want to expose properties of int range type. This function is typically + * used in connection with g_object_class_install_property() in a GObjects's + * instance_init function. + * + * Returns: (transfer full): a newly created parameter specification + */ +GParamSpec * +gst_param_spec_int_range (const gchar * name, const gchar * nick, const gchar * blurb, + gint min_min, gint min_max, gint min_step, + gint max_min, gint max_max, gint max_step, + gint def_min, gint def_max, gint def_step, + GParamFlags flags) +{ + GstParamSpecIntRange *irspec; + GParamSpec *pspec; + GValue default_val = G_VALUE_INIT; + + irspec = + g_param_spec_internal (GST_TYPE_PARAM_INT_RANGE, name, nick, blurb, flags); + + irspec->min_min = min_min; + irspec->min_max = min_max; + irspec->min_step = min_step; + irspec->max_min = max_min; + irspec->max_max = max_max; + irspec->max_step = max_step; + irspec->def_min = def_min; + irspec->def_max = def_max; + irspec->def_step = def_step; + + pspec = G_PARAM_SPEC (irspec); + + /* check that min <= default <= max */ + g_value_init (&default_val, GST_TYPE_INT_RANGE); + gst_value_set_int_range_step (&default_val, def_min, def_max, def_step); + + if (_gst_param_int_range_validate (pspec, &default_val)) { + g_critical ("GstParamSpec of type 'int_range' for property '%s' has a " + "default value of %d->%d (%d), which is not within the allowed range of " + "%d->%d to %d->%d", + name, def_min, def_max, def_step, min_min, max_min, min_max, max_max); + g_param_spec_ref (pspec); + g_param_spec_sink (pspec); + g_param_spec_unref (pspec); + pspec = NULL; + } + g_value_unset (&default_val); + + return pspec; +} diff --git a/subprojects/gstreamer/gst/gstparamspecs.h b/subprojects/gstreamer/gst/gstparamspecs.h index ff35507c737..672dbdb1e15 100644 --- a/subprojects/gstreamer/gst/gstparamspecs.h +++ b/subprojects/gstreamer/gst/gstparamspecs.h @@ -117,6 +117,9 @@ G_BEGIN_DECLS #define GST_IS_PARAM_SPEC_ARRAY_LIST(pspec) (G_TYPE_CHECK_INSTANCE_TYPE ((pspec), GST_TYPE_PARAM_ARRAY_LIST)) #define GST_PARAM_SPEC_ARRAY_LIST(pspec) (G_TYPE_CHECK_INSTANCE_CAST ((pspec), GST_TYPE_PARAM_ARRAY_LIST, GstParamSpecArray)) +#define GST_TYPE_PARAM_INT_RANGE (gst_param_spec_int_range_get_type ()) +#define GST_IS_PARAM_SPEC_INT_RANGE(pspec) (G_TYPE_CHECK_INSTANCE_TYPE ((pspec), GST_TYPE_PARAM_INT_RANGE)) +#define GST_PARAM_SPEC_INT_RANGE(pspec) (G_TYPE_CHECK_INSTANCE_CAST ((pspec), GST_TYPE_PARAM_INT_RANGE, GstParamSpecIntRange)) /* --- get_type functions --- */ @@ -126,11 +129,15 @@ GType gst_param_spec_fraction_get_type (void); GST_API GType gst_param_spec_array_get_type (void); +GST_API +GType gst_param_spec_int_range_get_type (void); + /* --- typedefs & structures --- */ typedef struct _GstParamSpecFraction GstParamSpecFraction; typedef struct _GstParamSpecArray GstParamSpecArray; +typedef struct _GstParamSpecIntRange GstParamSpecIntRange; /** * GstParamSpecFraction: @@ -166,6 +173,29 @@ struct _GstParamSpecArray { GParamSpec * element_spec; }; +/** + * GstParamSpecIntRange: + * @parent_instance: super class + * @min_min: minimal lowest value in range + * @min_max: minimal hightst value in range + * @min_step: minimal step + * @max_min: maximal lowest value in range + * @max_max: maximal hightst value in range + * @max_step: maximal step + * @def_min: default lowest value in range + * @def_max: default hightst value in range + * @def_step: default step + * + * A GParamSpec derived structure that contains the meta data for int range + * properties. + */ +struct _GstParamSpecIntRange { + GParamSpec parent_instance; + + gint min_min, min_max, min_step; + gint max_min, max_max, max_step; + gint def_min, def_max, def_step; +}; /* --- GParamSpec prototypes --- */ @@ -184,6 +214,16 @@ GParamSpec * gst_param_spec_array (const gchar * name, GParamSpec * element_spec, GParamFlags flags) G_GNUC_MALLOC; +GST_API +GParamSpec * gst_param_spec_int_range (const gchar * name, + const gchar * nick, + const gchar * blurb, + gint min_min, gint min_max, gint min_step, + gint max_min, gint max_max, gint max_step, + gint def_min, gint def_max, gint def_step, + GParamFlags flags) G_GNUC_MALLOC; + + G_END_DECLS #endif /* __GST_PARAMSPECS_H__ */ diff --git a/subprojects/gstreamer/gst/gstquark.c b/subprojects/gstreamer/gst/gstquark.c index 304e274155d..3138d9eea44 100644 --- a/subprojects/gstreamer/gst/gstquark.c +++ b/subprojects/gstreamer/gst/gstquark.c @@ -80,7 +80,7 @@ static const gchar *_quark_strings[] = { "GstEventInstantRateChange", "GstEventInstantRateSyncTime", "GstMessageInstantRateRequest", "upstream-running-time", "base", "offset", "plugin-api", "plugin-api-flags", - "gap-flags", "GstQuerySelectable", "selectable" + "gap-flags", "GstQuerySelectable", "selectable", "latency-changed" }; GQuark _priv_gst_quark_table[GST_QUARK_MAX]; diff --git a/subprojects/gstreamer/gst/gstquark.h b/subprojects/gstreamer/gst/gstquark.h index f7de510c9b1..ecb27db6314 100644 --- a/subprojects/gstreamer/gst/gstquark.h +++ b/subprojects/gstreamer/gst/gstquark.h @@ -233,7 +233,8 @@ typedef enum _GstQuarkId GST_QUARK_GAP_FLAGS = 202, GST_QUARK_QUERY_SELECTABLE = 203, GST_QUARK_SELECTABLE = 204, - GST_QUARK_MAX = 205 + GST_QUARK_EVENT_LATENCY_CHANGED = 205, + GST_QUARK_MAX = 206 } GstQuarkId; extern GQuark _priv_gst_quark_table[GST_QUARK_MAX]; diff --git a/subprojects/gstreamer/gst/gstsystemclock.c b/subprojects/gstreamer/gst/gstsystemclock.c index 827a7e1b565..fcc85831210 100644 --- a/subprojects/gstreamer/gst/gstsystemclock.c +++ b/subprojects/gstreamer/gst/gstsystemclock.c @@ -47,6 +47,10 @@ #include +#ifdef HAVE_PTHREAD_H +#include +#endif + #ifdef G_OS_WIN32 # define WIN32_LEAN_AND_MEAN /* prevents from including too many things */ # include /* QueryPerformance* stuff */ diff --git a/subprojects/gstreamer/gst/gsttask.c b/subprojects/gstreamer/gst/gsttask.c index c1356450844..4e94f1a633f 100644 --- a/subprojects/gstreamer/gst/gsttask.c +++ b/subprojects/gstreamer/gst/gsttask.c @@ -230,8 +230,17 @@ G_DEFINE_TYPE_WITH_CODE (GstTask, gst_task, GST_TYPE_OBJECT, static void ensure_klass_pool (GstTaskClass * klass) { - if (G_UNLIKELY (_global_task_pool == NULL)) { - _global_task_pool = gst_task_pool_new (); + if (_global_task_pool) { + if (G_TYPE_FROM_INSTANCE (_global_task_pool) != klass->pool_type) { + GST_INFO ("task pool type changed. Recreating global pool"); + gst_task_pool_cleanup (_global_task_pool); + gst_object_unref (_global_task_pool); + _global_task_pool = NULL; + } + } + + if (_global_task_pool == NULL) { + _global_task_pool = g_object_new (klass->pool_type, NULL); gst_task_pool_prepare (_global_task_pool, NULL); /* Classes are never destroyed so this ref will never be dropped */ @@ -248,6 +257,17 @@ gst_task_class_init (GstTaskClass * klass) gobject_class = (GObjectClass *) klass; gobject_class->finalize = gst_task_finalize; + + klass->pool_type = GST_TYPE_TASK_POOL; +} + +void +gst_task_class_set_default_task_pool_type (GType type) +{ + GstTaskClass *klass = g_type_class_peek (gst_task_get_type ()); + g_mutex_lock (&pool_lock); + klass->pool_type = type; + g_mutex_unlock (&pool_lock); } static void diff --git a/subprojects/gstreamer/gst/gsttask.h b/subprojects/gstreamer/gst/gsttask.h index 7ae1d43abbb..000f9ce70b0 100644 --- a/subprojects/gstreamer/gst/gsttask.h +++ b/subprojects/gstreamer/gst/gsttask.h @@ -159,6 +159,7 @@ struct _GstTaskClass { /*< private >*/ GstTaskPool *pool; + GType pool_type; /*< private >*/ gpointer _gst_reserved[GST_PADDING]; @@ -213,6 +214,9 @@ gboolean gst_task_resume (GstTask *task); GST_API gboolean gst_task_join (GstTask *task); +GST_API +void gst_task_class_set_default_task_pool_type (GType type); + G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstTask, gst_object_unref) G_END_DECLS diff --git a/subprojects/gstreamer/libs/gst/base/base.h b/subprojects/gstreamer/libs/gst/base/base.h index 5ad42c11b27..fcb99feb425 100644 --- a/subprojects/gstreamer/libs/gst/base/base.h +++ b/subprojects/gstreamer/libs/gst/base/base.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -37,6 +38,7 @@ #include #include #include +#include #include #include #include diff --git a/subprojects/gstreamer/libs/gst/base/gstaggregator.c b/subprojects/gstreamer/libs/gst/base/gstaggregator.c index a6aa43af0cb..833f7ac847c 100644 --- a/subprojects/gstreamer/libs/gst/base/gstaggregator.c +++ b/subprojects/gstreamer/libs/gst/base/gstaggregator.c @@ -2267,7 +2267,7 @@ gst_aggregator_query_latency_unlocked (GstAggregator * self, GstQuery * query) SRC_LOCK (self); if (!query_ret) { - GST_WARNING_OBJECT (self, "Latency query failed"); + GST_INFO_OBJECT (self, "Latency query failed"); return FALSE; } diff --git a/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.c b/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.c new file mode 100644 index 00000000000..01192fc82fe --- /dev/null +++ b/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.c @@ -0,0 +1,1688 @@ +/* GStreamer + * Copyright (C) 1999,2000 Erik Walthinsen + * 2000,2005 Wim Taymans + * 2023 Havard Graff + * 2023 Camilo Celis Guzman + * + * gstbaseidlesrc.c: + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:gstbaseidlesrc + * @title: GstBaseIdleSrc + * @short_description: Base class for getrange based source elements + * @see_also: #GstPushSrc, #GstBaseTransform, #GstBaseSink + * + + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include +#include + +#include +#include + +#include "gstbaseidlesrc.h" + +GST_DEBUG_CATEGORY_STATIC (gst_base_idle_src_debug); +#define GST_CAT_DEFAULT gst_base_idle_src_debug + +/* BaseIdleSrc signals and args */ +enum +{ + /* FILL ME */ + LAST_SIGNAL +}; + +#define DEFAULT_DO_TIMESTAMP FALSE + +enum +{ + PROP_0, + PROP_DO_TIMESTAMP +}; + +/* The src implementation need to respect the following locking order: + * 1. STREAM_LOCK + * 2. LIVE_LOCK + * 3. OBJECT_LOCK + */ +struct _GstBaseIdleSrcPrivate +{ + /* if a stream-start event should be sent */ + gboolean stream_start_pending; /* STREAM_LOCK */ + + /* if segment should be sent and a + * seqnum if it was originated by a seek */ + gboolean segment_pending; /* OBJECT_LOCK */ + guint32 segment_seqnum; /* OBJECT_LOCK */ + + /* startup latency is the time it takes between going to PLAYING and producing + * the first BUFFER with running_time 0. This value is included in the latency + * reporting. */ + GstClockTime latency; /* OBJECT_LOCK */ + /* timestamp offset, this is the offset add to the values of gst_times for + * pseudo live sources */ + GstClockTimeDiff ts_offset; /* OBJECT_LOCK */ + + gboolean do_timestamp; /* OBJECT_LOCK */ + + /* QoS *//* with LOCK */ + gdouble proportion; /* OBJECT_LOCK */ + GstClockTime earliest_time; /* OBJECT_LOCK */ + + GstBufferPool *pool; /* OBJECT_LOCK */ + GstAllocator *allocator; /* OBJECT_LOCK */ + GstAllocationParams params; /* OBJECT_LOCK */ + + GQueue *obj_queue; + GThread *thread; +}; + +static GstElementClass *parent_class = NULL; +static gint private_offset = 0; + + +static void gst_base_idle_src_start_task (GstBaseIdleSrc * src, gboolean wait); +static void gst_base_idle_src_process_object_queue (GstBaseIdleSrc * src); + +static inline GstBaseIdleSrcPrivate * +gst_base_idle_src_get_instance_private (GstBaseIdleSrc * self) +{ + return (G_STRUCT_MEMBER_P (self, private_offset)); +} + +/* TODO: do we support anything other than _BUFFER/_BYTES ? */ +/** + * gst_base_idle_src_set_format: + * @src: base source instance + * @format: the format to use + * + * Sets the default format of the source. This will be the format used + * for sending SEGMENT events and for performing seeks. + * + * If a format of GST_FORMAT_BYTES is set, the element will be able to + * operate in pull mode if the #GstBaseIdleSrcClass::is_seekable returns %TRUE. + * + * This function must only be called in states < %GST_STATE_PAUSED. + */ +void +gst_base_idle_src_set_format (GstBaseIdleSrc * src, GstFormat format) +{ + g_return_if_fail (GST_IS_BASE_IDLE_SRC (src)); + g_return_if_fail (GST_STATE (src) <= GST_STATE_READY); + + GST_OBJECT_LOCK (src); + gst_segment_init (&src->segment, format); + GST_OBJECT_UNLOCK (src); +} + +/** + * gst_base_idle_src_query_latency: + * @src: the source + * @live: (out) (allow-none): if the source is live + * @min_latency: (out) (allow-none): the min latency of the source + * @max_latency: (out) (allow-none): the max latency of the source + * + * Query the source for the latency parameters. @live will be %TRUE when @src is + * configured as a live source. @min_latency and @max_latency will be set + * to the difference between the running time and the timestamp of the first + * buffer. + * + * This function is mostly used by subclasses. + * + * Returns: %TRUE if the query succeeded. + */ +gboolean +gst_base_idle_src_query_latency (GstBaseIdleSrc * src, gboolean * live, + GstClockTime * min_latency, GstClockTime * max_latency) +{ + GstClockTime min; + + g_return_val_if_fail (GST_IS_BASE_IDLE_SRC (src), FALSE); + + GST_OBJECT_LOCK (src); + if (live) + *live = src->is_live; + + /* if we have a startup latency, report this one, else report 0. Subclasses + * are supposed to override the query function if they want something + * else. */ + if (src->priv->latency != -1) + min = src->priv->latency; + else + min = 0; + + if (min_latency) + *min_latency = min; + if (max_latency) + *max_latency = min; + + GST_LOG_OBJECT (src, "latency: live %d, min %" GST_TIME_FORMAT + ", max %" GST_TIME_FORMAT, src->is_live, GST_TIME_ARGS (min), + GST_TIME_ARGS (min)); + GST_OBJECT_UNLOCK (src); + + return TRUE; +} + +/** + * gst_base_idle_src_set_do_timestamp: + * @src: the source + * @timestamp: enable or disable timestamping + * + * Configure @src to automatically timestamp outgoing buffers based on the + * current running_time of the pipeline. This property is mostly useful for live + * sources. + */ +void +gst_base_idle_src_set_do_timestamp (GstBaseIdleSrc * src, gboolean timestamp) +{ + g_return_if_fail (GST_IS_BASE_IDLE_SRC (src)); + + GST_OBJECT_LOCK (src); + src->priv->do_timestamp = timestamp; + if (timestamp && src->segment.format != GST_FORMAT_TIME) + gst_segment_init (&src->segment, GST_FORMAT_TIME); + GST_OBJECT_UNLOCK (src); +} + +/** + * gst_base_idle_src_get_do_timestamp: + * @src: the source + * + * Query if @src timestamps outgoing buffers based on the current running_time. + * + * Returns: %TRUE if the base class will automatically timestamp outgoing buffers. + */ +gboolean +gst_base_idle_src_get_do_timestamp (GstBaseIdleSrc * src) +{ + gboolean res; + + g_return_val_if_fail (GST_IS_BASE_IDLE_SRC (src), FALSE); + + GST_OBJECT_LOCK (src); + res = src->priv->do_timestamp; + GST_OBJECT_UNLOCK (src); + + return res; +} + +/** + * gst_base_idle_src_new_segment: + * @src: a #GstBaseIdleSrc + * @segment: a pointer to a #GstSegment + * + * Prepare a new segment for emission downstream. This function must + * only be called by derived sub-classes, and only from the #GstBaseIdleSrcClass::create function, + * as the stream-lock needs to be held. + * + * The format for the @segment must be identical with the current format + * of the source, as configured with gst_base_idle_src_set_format(). + * + * The format of @src must not be %GST_FORMAT_UNDEFINED and the format + * should be configured via gst_base_idle_src_set_format() before calling this method. + * + * Returns: %TRUE if preparation of new segment succeeded. + * + * Since: 1.18 + */ +gboolean +gst_base_idle_src_new_segment (GstBaseIdleSrc * src, const GstSegment * segment) +{ + g_return_val_if_fail (GST_IS_BASE_IDLE_SRC (src), FALSE); + g_return_val_if_fail (segment != NULL, FALSE); + + GST_OBJECT_LOCK (src); + + if (src->segment.format == GST_FORMAT_UNDEFINED) { + /* subclass must set valid format before calling this method */ + GST_WARNING_OBJECT (src, "segment format is not configured yet, ignore"); + GST_OBJECT_UNLOCK (src); + return FALSE; + } + + if (src->segment.format != segment->format) { + GST_WARNING_OBJECT (src, "segment format mismatched, ignore"); + GST_OBJECT_UNLOCK (src); + return FALSE; + } + + gst_segment_copy_into (segment, &src->segment); + + /* Mark pending segment. Will be sent before next data */ + src->priv->segment_pending = TRUE; + src->priv->segment_seqnum = gst_util_seqnum_next (); + + GST_DEBUG_OBJECT (src, "Starting new segment %" GST_SEGMENT_FORMAT, segment); + + GST_OBJECT_UNLOCK (src); + + return TRUE; +} + +static void +gst_base_idle_src_queue_object (GstBaseIdleSrc * src, GstMiniObject * obj) +{ + GST_LOG_OBJECT (src, "Queuing: %" GST_PTR_FORMAT, obj); + GST_OBJECT_LOCK (src); + g_queue_push_tail (src->priv->obj_queue, obj); + GST_OBJECT_UNLOCK (src); +} + +/** + * gst_base_idle_src_set_caps: + * @src: a #GstBaseIdleSrc + * @caps: (transfer none): a #GstCaps + * + * Set new caps on the src source pad. + * This *MUST* be called before submitting any buffers! + * + * Returns: %TRUE if the caps could be set + */ +gboolean +gst_base_idle_src_set_caps (GstBaseIdleSrc * src, GstCaps * caps) +{ + GstBaseIdleSrcClass *bclass; + gboolean res = TRUE; + GstCaps *current_caps; + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + + current_caps = gst_pad_get_current_caps (GST_BASE_IDLE_SRC_PAD (src)); + if (current_caps && gst_caps_is_equal (current_caps, caps)) { + GST_DEBUG_OBJECT (src, "New caps equal to old ones: %" GST_PTR_FORMAT, + caps); + res = TRUE; + } else { + if (bclass->set_caps) + res = bclass->set_caps (src, caps); + + if (res) { + gst_base_idle_src_queue_object (src, + (GstMiniObject *) gst_event_new_caps (caps)); + gst_base_idle_src_process_object_queue (src); + } + } + + if (current_caps) + gst_caps_unref (current_caps); + + return res; +} + +static GstCaps * +gst_base_idle_src_default_get_caps (GstBaseIdleSrc * bsrc, GstCaps * filter) +{ + GstCaps *caps = NULL; + GstPadTemplate *pad_template; + GstBaseIdleSrcClass *bclass; + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (bsrc); + + pad_template = + gst_element_class_get_pad_template (GST_ELEMENT_CLASS (bclass), "src"); + + if (pad_template != NULL) { + caps = gst_pad_template_get_caps (pad_template); + + if (filter) { + GstCaps *intersection; + + intersection = + gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (caps); + caps = intersection; + } + } + return caps; +} + +static GstCaps * +gst_base_idle_src_default_fixate (GstBaseIdleSrc * bsrc, GstCaps * caps) +{ + GST_DEBUG_OBJECT (bsrc, "using default caps fixate function"); + return gst_caps_fixate (caps); +} + +static GstCaps * +gst_base_idle_src_fixate (GstBaseIdleSrc * bsrc, GstCaps * caps) +{ + GstBaseIdleSrcClass *bclass; + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (bsrc); + + if (bclass->fixate) + caps = bclass->fixate (bsrc, caps); + + return caps; +} + +static gboolean +gst_base_idle_src_default_query (GstBaseIdleSrc * src, GstQuery * query) +{ + gboolean res; + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_POSITION: + { + GstFormat format; + + gst_query_parse_position (query, &format, NULL); + + GST_DEBUG_OBJECT (src, "position query in format %s", + gst_format_get_name (format)); + + switch (format) { + case GST_FORMAT_PERCENT: + { + gint64 percent; + gint64 position; + gint64 duration; + + GST_OBJECT_LOCK (src); + position = src->segment.position; + duration = src->segment.duration; + GST_OBJECT_UNLOCK (src); + + if (position != -1 && duration != -1) { + if (position < duration) + percent = gst_util_uint64_scale (GST_FORMAT_PERCENT_MAX, position, + duration); + else + percent = GST_FORMAT_PERCENT_MAX; + } else + percent = -1; + + gst_query_set_position (query, GST_FORMAT_PERCENT, percent); + res = TRUE; + break; + } + default: + { + gint64 position; + GstFormat seg_format; + + GST_OBJECT_LOCK (src); + position = + gst_segment_to_stream_time (&src->segment, src->segment.format, + src->segment.position); + seg_format = src->segment.format; + GST_OBJECT_UNLOCK (src); + + if (position != -1) { + /* convert to requested format */ + res = + gst_pad_query_convert (src->srcpad, seg_format, + position, format, &position); + } else + res = TRUE; + + if (res) + gst_query_set_position (query, format, position); + + break; + } + } + break; + } + + case GST_QUERY_SEGMENT: + { + GstFormat format; + gint64 start, stop; + + GST_OBJECT_LOCK (src); + + format = src->segment.format; + + start = + gst_segment_to_stream_time (&src->segment, format, + src->segment.start); + if ((stop = src->segment.stop) == -1) + stop = src->segment.duration; + else + stop = gst_segment_to_stream_time (&src->segment, format, stop); + + gst_query_set_segment (query, src->segment.rate, format, start, stop); + + GST_OBJECT_UNLOCK (src); + res = TRUE; + break; + } + + case GST_QUERY_FORMATS: + { + gst_query_set_formats (query, 3, GST_FORMAT_DEFAULT, + GST_FORMAT_BYTES, GST_FORMAT_PERCENT); + res = TRUE; + break; + } + case GST_QUERY_CONVERT: + { + GstFormat src_fmt, dest_fmt; + gint64 src_val, dest_val; + + gst_query_parse_convert (query, &src_fmt, &src_val, &dest_fmt, &dest_val); + + /* we can only convert between equal formats... */ + if (src_fmt == dest_fmt) { + dest_val = src_val; + res = TRUE; + } else + res = FALSE; + + gst_query_set_convert (query, src_fmt, src_val, dest_fmt, dest_val); + break; + } + case GST_QUERY_LATENCY: + { + GstClockTime min, max; + gboolean live; + + /* Subclasses should override and implement something useful */ + res = gst_base_idle_src_query_latency (src, &live, &min, &max); + + GST_LOG_OBJECT (src, "report latency: live %d, min %" GST_TIME_FORMAT + ", max %" GST_TIME_FORMAT, live, GST_TIME_ARGS (min), + GST_TIME_ARGS (max)); + + gst_query_set_latency (query, live, min, max); + break; + } + case GST_QUERY_JITTER: + case GST_QUERY_RATE: + res = FALSE; + break; + case GST_QUERY_BUFFERING: + { + GstFormat format, seg_format; + gint64 start, stop, estimated; + + gst_query_parse_buffering_range (query, &format, NULL, NULL, NULL); + + GST_DEBUG_OBJECT (src, "buffering query in format %s", + gst_format_get_name (format)); + + GST_OBJECT_LOCK (src); + estimated = -1; + start = -1; + stop = -1; + seg_format = src->segment.format; + GST_OBJECT_UNLOCK (src); + + /* convert to required format. When the conversion fails, we can't answer + * the query. When the value is unknown, we can don't perform conversion + * but report TRUE. */ + if (format != GST_FORMAT_PERCENT && stop != -1) { + res = gst_pad_query_convert (src->srcpad, seg_format, + stop, format, &stop); + } else { + res = TRUE; + } + if (res && format != GST_FORMAT_PERCENT && start != -1) + res = gst_pad_query_convert (src->srcpad, seg_format, + start, format, &start); + + gst_query_set_buffering_range (query, format, start, stop, estimated); + break; + } + case GST_QUERY_CAPS: + { + GstBaseIdleSrcClass *bclass; + GstCaps *caps, *filter; + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + if (bclass->get_caps) { + gst_query_parse_caps (query, &filter); + if ((caps = bclass->get_caps (src, filter))) { + gst_query_set_caps_result (query, caps); + gst_caps_unref (caps); + res = TRUE; + } else { + res = FALSE; + } + } else + res = FALSE; + break; + } + case GST_QUERY_URI:{ + if (GST_IS_URI_HANDLER (src)) { + gchar *uri = gst_uri_handler_get_uri (GST_URI_HANDLER (src)); + + if (uri != NULL) { + gst_query_set_uri (query, uri); + g_free (uri); + res = TRUE; + } else { + res = FALSE; + } + } else { + res = FALSE; + } + break; + } + default: + res = FALSE; + break; + } + GST_DEBUG_OBJECT (src, "query %s returns %d", GST_QUERY_TYPE_NAME (query), + res); + + return res; +} + +static gboolean +gst_base_idle_src_query (GstPad * pad, GstObject * parent, GstQuery * query) +{ + GstBaseIdleSrc *src; + GstBaseIdleSrcClass *bclass; + gboolean result = FALSE; + + src = GST_BASE_IDLE_SRC (parent); + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + + if (bclass->query) + result = bclass->query (src, query); + + return result; +} + +static void +gst_base_idle_src_set_pool_flushing (GstBaseIdleSrc * src, gboolean flushing) +{ + GstBaseIdleSrcPrivate *priv = src->priv; + GstBufferPool *pool; + + GST_OBJECT_LOCK (src); + if ((pool = priv->pool)) + pool = gst_object_ref (pool); + GST_OBJECT_UNLOCK (src); + + if (pool) { + gst_buffer_pool_set_flushing (pool, flushing); + gst_object_unref (pool); + } +} + +/* all events send to this element directly. This is mainly done from the + * application. + */ +static gboolean +gst_base_idle_src_send_event (GstElement * element, GstEvent * event) +{ + GstBaseIdleSrc *src; + gboolean result = FALSE; + + src = GST_BASE_IDLE_SRC (element); + + GST_DEBUG_OBJECT (src, "handling event %p %" GST_PTR_FORMAT, event, event); + + switch (GST_EVENT_TYPE (event)) { + /* downstream serialized events */ + case GST_EVENT_EOS: + { + gst_base_idle_src_set_pool_flushing (src, TRUE); + event = NULL; + result = TRUE; + break; + } + case GST_EVENT_SEGMENT: + /* sending random SEGMENT downstream can break sync. */ + break; + case GST_EVENT_TAG: + case GST_EVENT_SINK_MESSAGE: + case GST_EVENT_CUSTOM_DOWNSTREAM: + case GST_EVENT_CUSTOM_BOTH: + case GST_EVENT_PROTECTION: + /* Insert TAG, CUSTOM_DOWNSTREAM, CUSTOM_BOTH, PROTECTION in the dataflow */ + gst_base_idle_src_queue_object (src, (GstMiniObject *) event); + event = NULL; + result = TRUE; + break; + case GST_EVENT_BUFFERSIZE: + /* does not seem to make much sense currently */ + break; + + /* upstream events */ + case GST_EVENT_QOS: + /* elements should override send_event and do something */ + break; + case GST_EVENT_NAVIGATION: + /* could make sense for elements that do something with navigation events + * but then they would need to override the send_event function */ + break; + case GST_EVENT_LATENCY: + /* does not seem to make sense currently */ + break; + + /* custom events */ + case GST_EVENT_CUSTOM_UPSTREAM: + /* override send_event if you want this */ + break; + case GST_EVENT_CUSTOM_DOWNSTREAM_OOB: + case GST_EVENT_CUSTOM_BOTH_OOB: + /* insert a random custom event into the pipeline */ + GST_DEBUG_OBJECT (src, "pushing custom OOB event downstream"); + result = gst_pad_push_event (src->srcpad, event); + /* we gave away the ref to the event in the push */ + event = NULL; + break; + default: + break; + } + + + return result; +} + +static void +gst_base_idle_src_update_qos (GstBaseIdleSrc * src, + gdouble proportion, GstClockTimeDiff diff, GstClockTime timestamp) +{ + GST_CAT_DEBUG_OBJECT (GST_CAT_QOS, src, + "qos: proportion: %lf, diff %" G_GINT64_FORMAT ", timestamp %" + GST_TIME_FORMAT, proportion, diff, GST_TIME_ARGS (timestamp)); + + GST_OBJECT_LOCK (src); + src->priv->proportion = proportion; + src->priv->earliest_time = timestamp + diff; + GST_OBJECT_UNLOCK (src); +} + + +static gboolean +gst_base_idle_src_default_event (GstBaseIdleSrc * src, GstEvent * event) +{ + gboolean result; + + GST_DEBUG_OBJECT (src, "handle event %" GST_PTR_FORMAT, event); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_QOS: + { + gdouble proportion; + GstClockTimeDiff diff; + GstClockTime timestamp; + + gst_event_parse_qos (event, NULL, &proportion, &diff, ×tamp); + gst_base_idle_src_update_qos (src, proportion, diff, timestamp); + result = TRUE; + break; + } + case GST_EVENT_RECONFIGURE: + result = TRUE; + break; + case GST_EVENT_LATENCY: + result = TRUE; + break; + default: + result = FALSE; + break; + } + return result; +} + +static gboolean +gst_base_idle_src_event (GstPad * pad, GstObject * parent, GstEvent * event) +{ + GstBaseIdleSrc *src; + GstBaseIdleSrcClass *bclass; + gboolean result = FALSE; + + src = GST_BASE_IDLE_SRC (parent); + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + + if (bclass->event) { + if (!(result = bclass->event (src, event))) + goto subclass_failed; + } + +done: + gst_event_unref (event); + + return result; + + /* ERRORS */ +subclass_failed: + { + GST_DEBUG_OBJECT (src, "subclass refused event"); + goto done; + } +} + +static void +gst_base_idle_src_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstBaseIdleSrc *src; + + src = GST_BASE_IDLE_SRC (object); + + switch (prop_id) { + case PROP_DO_TIMESTAMP: + gst_base_idle_src_set_do_timestamp (src, g_value_get_boolean (value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_base_idle_src_get_property (GObject * object, guint prop_id, GValue * value, + GParamSpec * pspec) +{ + GstBaseIdleSrc *src; + + src = GST_BASE_IDLE_SRC (object); + + switch (prop_id) { + case PROP_DO_TIMESTAMP: + g_value_set_boolean (value, gst_base_idle_src_get_do_timestamp (src)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +gst_base_idle_src_set_allocation (GstBaseIdleSrc * src, + GstBufferPool * pool, GstAllocator * allocator, + const GstAllocationParams * params) +{ + GstAllocator *oldalloc; + GstBufferPool *oldpool; + GstBaseIdleSrcPrivate *priv = src->priv; + + if (pool) { + GST_DEBUG_OBJECT (src, "activate pool"); + if (!gst_buffer_pool_set_active (pool, TRUE)) + goto activate_failed; + } + + GST_OBJECT_LOCK (src); + oldpool = priv->pool; + priv->pool = pool; + + oldalloc = priv->allocator; + priv->allocator = allocator; + + if (priv->pool) + gst_object_ref (priv->pool); + if (priv->allocator) + gst_object_ref (priv->allocator); + + if (params) + priv->params = *params; + else + gst_allocation_params_init (&priv->params); + GST_OBJECT_UNLOCK (src); + + if (oldpool) { + /* only deactivate if the pool is not the one we're using */ + if (oldpool != pool) { + GST_DEBUG_OBJECT (src, "deactivate old pool"); + gst_buffer_pool_set_active (oldpool, FALSE); + } + gst_object_unref (oldpool); + } + if (oldalloc) { + gst_object_unref (oldalloc); + } + return TRUE; + + /* ERRORS */ +activate_failed: + { + GST_ERROR_OBJECT (src, "failed to activate bufferpool."); + return FALSE; + } +} + +static gboolean +gst_base_idle_src_decide_allocation_default (GstBaseIdleSrc * src, + GstQuery * query) +{ + GstCaps *outcaps; + GstBufferPool *pool; + guint size, min, max; + GstAllocator *allocator; + GstAllocationParams params; + GstStructure *config; + gboolean update_allocator; + + gst_query_parse_allocation (query, &outcaps, NULL); + + /* we got configuration from our peer or the decide_allocation method, + * parse them */ + if (gst_query_get_n_allocation_params (query) > 0) { + /* try the allocator */ + gst_query_parse_nth_allocation_param (query, 0, &allocator, ¶ms); + update_allocator = TRUE; + } else { + allocator = NULL; + gst_allocation_params_init (¶ms); + update_allocator = FALSE; + } + + if (gst_query_get_n_allocation_pools (query) > 0) { + gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); + + if (pool == NULL) { + /* no pool, we can make our own */ + GST_DEBUG_OBJECT (src, "no pool, making new pool"); + pool = gst_buffer_pool_new (); + } + } else { + pool = NULL; + size = min = max = 0; + } + + /* now configure */ + if (pool) { + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_set_params (config, outcaps, size, min, max); + gst_buffer_pool_config_set_allocator (config, allocator, ¶ms); + + /* buffer pool may have to do some changes */ + if (!gst_buffer_pool_set_config (pool, config)) { + config = gst_buffer_pool_get_config (pool); + + /* If change are not acceptable, fallback to generic pool */ + if (!gst_buffer_pool_config_validate_params (config, outcaps, size, min, + max)) { + GST_DEBUG_OBJECT (src, "unsupported pool, making new pool"); + + gst_object_unref (pool); + pool = gst_buffer_pool_new (); + gst_buffer_pool_config_set_params (config, outcaps, size, min, max); + gst_buffer_pool_config_set_allocator (config, allocator, ¶ms); + } + + if (!gst_buffer_pool_set_config (pool, config)) + goto config_failed; + } + } + + if (update_allocator) + gst_query_set_nth_allocation_param (query, 0, allocator, ¶ms); + else + gst_query_add_allocation_param (query, allocator, ¶ms); + if (allocator) + gst_object_unref (allocator); + + if (pool) { + gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); + gst_object_unref (pool); + } + + return TRUE; + +config_failed: + GST_ELEMENT_ERROR (src, RESOURCE, SETTINGS, + ("Failed to configure the buffer pool"), + ("Configuration is most likely invalid, please report this issue.")); + gst_object_unref (pool); + return FALSE; +} + +static gboolean +gst_base_idle_src_prepare_allocation (GstBaseIdleSrc * src, GstCaps * caps) +{ + GstBaseIdleSrcClass *bclass; + gboolean result = TRUE; + GstQuery *query; + GstBufferPool *pool = NULL; + GstAllocator *allocator = NULL; + GstAllocationParams params; + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + + /* make query and let peer pad answer, we don't really care if it worked or + * not, if it failed, the allocation query would contain defaults and the + * subclass would then set better values if needed */ + query = gst_query_new_allocation (caps, TRUE); + if (!gst_pad_peer_query (src->srcpad, query)) { + /* not a problem, just debug a little */ + GST_DEBUG_OBJECT (src, "peer ALLOCATION query failed"); + } + + g_assert (bclass->decide_allocation != NULL); + result = bclass->decide_allocation (src, query); + + GST_DEBUG_OBJECT (src, "ALLOCATION (%d) params: %" GST_PTR_FORMAT, result, + query); + + if (!result) + goto no_decide_allocation; + + /* we got configuration from our peer or the decide_allocation method, + * parse them */ + if (gst_query_get_n_allocation_params (query) > 0) { + gst_query_parse_nth_allocation_param (query, 0, &allocator, ¶ms); + } else { + allocator = NULL; + gst_allocation_params_init (¶ms); + } + + if (gst_query_get_n_allocation_pools (query) > 0) + gst_query_parse_nth_allocation_pool (query, 0, &pool, NULL, NULL, NULL); + + result = gst_base_idle_src_set_allocation (src, pool, allocator, ¶ms); + + if (allocator) + gst_object_unref (allocator); + if (pool) + gst_object_unref (pool); + + gst_query_unref (query); + + return result; + + /* Errors */ +no_decide_allocation: + { + GST_WARNING_OBJECT (src, "Subclass failed to decide allocation"); + gst_query_unref (query); + + return result; + } +} + +/* default negotiation code. + * + * Take intersection between src and sink pads, take first + * caps and fixate. + + * only called from gst_base_idle_src_func + */ +static gboolean +gst_base_idle_src_default_negotiate (GstBaseIdleSrc * src) +{ + GstCaps *thiscaps; + GstCaps *caps = NULL; + GstCaps *peercaps = NULL; + gboolean result = FALSE; + + /* first see what is possible on our source pad */ + thiscaps = gst_pad_query_caps (GST_BASE_IDLE_SRC_PAD (src), NULL); + GST_DEBUG_OBJECT (src, "caps of src: %" GST_PTR_FORMAT, thiscaps); + /* nothing or anything is allowed, we're done */ + if (thiscaps == NULL || gst_caps_is_any (thiscaps)) + goto no_nego_needed; + + if (G_UNLIKELY (gst_caps_is_empty (thiscaps))) + goto no_caps; + + /* get the peer caps */ + peercaps = gst_pad_peer_query_caps (GST_BASE_IDLE_SRC_PAD (src), thiscaps); + GST_DEBUG_OBJECT (src, "caps of peer: %" GST_PTR_FORMAT, peercaps); + if (peercaps) { + /* The result is already a subset of our caps */ + caps = peercaps; + gst_caps_unref (thiscaps); + } else { + /* no peer, work with our own caps then */ + caps = thiscaps; + } + if (caps && !gst_caps_is_empty (caps)) { + /* now fixate */ + GST_DEBUG_OBJECT (src, "have caps: %" GST_PTR_FORMAT, caps); + if (gst_caps_is_any (caps)) { + GST_DEBUG_OBJECT (src, "any caps, we stop"); + /* hmm, still anything, so element can do anything and + * nego is not needed */ + result = TRUE; + } else { + caps = gst_base_idle_src_fixate (src, caps); + GST_DEBUG_OBJECT (src, "fixated to: %" GST_PTR_FORMAT, caps); + if (gst_caps_is_fixed (caps)) { + /* yay, fixed caps, use those then, it's possible that the subclass does + * not accept this caps after all and we have to fail. */ + result = gst_base_idle_src_set_caps (src, caps); + } + } + gst_caps_unref (caps); + } else { + if (caps) + gst_caps_unref (caps); + GST_DEBUG_OBJECT (src, "no common caps"); + } + return result; + +no_nego_needed: + { + GST_DEBUG_OBJECT (src, "no negotiation needed"); + if (thiscaps) + gst_caps_unref (thiscaps); + return TRUE; + } +no_caps: + { + GST_ELEMENT_ERROR (src, STREAM, FORMAT, + ("No supported formats found"), + ("This element did not produce valid caps")); + if (thiscaps) + gst_caps_unref (thiscaps); + return TRUE; + } +} + +/* only called from gst_base_idle_src_func */ +static gboolean +gst_base_idle_src_negotiate_unlocked (GstBaseIdleSrc * src) +{ + GstBaseIdleSrcClass *bclass; + gboolean result; + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + + GST_DEBUG_OBJECT (src, "starting negotiation"); + + if (G_LIKELY (bclass->negotiate)) + result = bclass->negotiate (src); + else + result = TRUE; + + if (G_LIKELY (result)) { + GstCaps *caps; + + caps = gst_pad_get_current_caps (src->srcpad); + + result = gst_base_idle_src_prepare_allocation (src, caps); + + if (caps) + gst_caps_unref (caps); + } + return result; +} + +/** + * gst_base_idle_src_get_buffer_pool: + * @src: a #GstBaseIdleSrc + * + * Returns: (nullable) (transfer full): the instance of the #GstBufferPool used + * by the src; unref it after usage. + */ +GstBufferPool * +gst_base_idle_src_get_buffer_pool (GstBaseIdleSrc * src) +{ + GstBufferPool *ret = NULL; + + g_return_val_if_fail (GST_IS_BASE_IDLE_SRC (src), NULL); + + GST_OBJECT_LOCK (src); + if (src->priv->pool) + ret = gst_object_ref (src->priv->pool); + GST_OBJECT_UNLOCK (src); + + return ret; +} + +/** + * gst_base_idle_src_get_allocator: + * @src: a #GstBaseIdleSrc + * @allocator: (out) (optional) (nullable) (transfer full): the #GstAllocator + * used + * @params: (out caller-allocates) (optional): the #GstAllocationParams of @allocator + * + * Lets #GstBaseIdleSrc sub-classes to know the memory @allocator + * used by the base class and its @params. + * + * Unref the @allocator after usage. + */ +void +gst_base_idle_src_get_allocator (GstBaseIdleSrc * src, + GstAllocator ** allocator, GstAllocationParams * params) +{ + g_return_if_fail (GST_IS_BASE_IDLE_SRC (src)); + + GST_OBJECT_LOCK (src); + if (allocator) + *allocator = src->priv->allocator ? + gst_object_ref (src->priv->allocator) : NULL; + + if (params) + *params = src->priv->params; + GST_OBJECT_UNLOCK (src); +} + +static void +gst_base_idle_src_add_timestamp (GstBaseIdleSrc * src, GstBuffer * buf) +{ + GstClock *clock; + GstClockTime base_time; + GstClockTime now; + GstClockTime running_time; + + if ((clock = GST_ELEMENT_CLOCK (src)) == NULL) { + GST_ERROR_OBJECT (src, "No clock!?! Can't do timestamp"); + return; + } else { + gst_object_ref (clock); + } + base_time = GST_ELEMENT_CAST (src)->base_time; + now = gst_clock_get_time (clock); + gst_object_unref (clock); + + running_time = now - base_time; + GST_BUFFER_PTS (buf) = running_time; + GST_BUFFER_DTS (buf) = running_time; +} + +static void +gst_base_idle_src_process_object (GstBaseIdleSrc * src, GstMiniObject * obj) +{ + GstPad *pad = src->srcpad; + + GST_PAD_STREAM_LOCK (pad); + + if (GST_IS_BUFFER (obj)) { + GstBuffer *buf = GST_BUFFER_CAST (obj); + GstFlowReturn ret; + + if (src->priv->do_timestamp) { + gst_base_idle_src_add_timestamp (src, buf); + } + + GST_DEBUG_OBJECT (src, "About to push Buffer %" GST_PTR_FORMAT, buf); + + ret = gst_pad_push (pad, buf); + if (ret != GST_FLOW_OK) + GST_ERROR ("Got ret: %s", gst_flow_get_name (ret)); + } else if (GST_IS_EVENT (obj)) { + GstEvent *event = GST_EVENT_CAST (obj); + gboolean ret; + + GST_DEBUG_OBJECT (src, "About to push Event %" GST_PTR_FORMAT, event); + ret = gst_pad_push_event (pad, event); + if (!ret) + GST_ERROR ("HUUUUAA"); + } else if (GST_IS_CAPS (obj)) { + GST_DEBUG_OBJECT (src, "About to push Caps %" GST_PTR_FORMAT, obj); + } + + GST_PAD_STREAM_UNLOCK (pad); +} + +static void +gst_base_idle_src_process_object_queue (GstBaseIdleSrc * src) +{ + GstMiniObject *obj; + GST_OBJECT_LOCK (src); + while ((obj = g_queue_pop_head (src->priv->obj_queue))) { + + GST_OBJECT_UNLOCK (src); + gst_base_idle_src_process_object (src, obj); + GST_OBJECT_LOCK (src); + } + GST_OBJECT_UNLOCK (src); +} + +static gpointer +gst_base_idle_src_func (gpointer user_data) +{ + GstBaseIdleSrc *src = GST_BASE_IDLE_SRC (user_data); + + GST_DEBUG_OBJECT (src, "Thread func started"); + + GST_PAD_STREAM_LOCK (src->srcpad); + if (gst_pad_check_reconfigure (src->srcpad)) { + if (!gst_base_idle_src_negotiate_unlocked (src)) { + GST_ELEMENT_FLOW_ERROR (src, GST_FLOW_NOT_NEGOTIATED); + gst_pad_push_event (src->srcpad, gst_event_new_eos ()); + GST_PAD_STREAM_UNLOCK (src->srcpad); + return NULL; + } + } + GST_PAD_STREAM_UNLOCK (src->srcpad); + + gst_base_idle_src_process_object_queue (src); + + return NULL; +} + +static void +gst_base_idle_src_start_task (GstBaseIdleSrc * src, gboolean wait) +{ + GST_DEBUG_OBJECT (src, "Starting Task"); + /* if we already have an outstanding task, join it */ + if (src->priv->thread) + g_thread_join (src->priv->thread); + + src->priv->thread = + g_thread_new (GST_ELEMENT_NAME (src), gst_base_idle_src_func, src); + if (wait) { + g_thread_join (src->priv->thread); + src->priv->thread = NULL; + } +} + +static void +gst_base_idle_src_check_pending_segment (GstBaseIdleSrc * src) +{ + GST_DEBUG_OBJECT (src, "Checking pending segment"); + /* push events to close/start our segment before we push the buffer. */ + if (G_UNLIKELY (src->priv->segment_pending)) { + GstEvent *seg_event = gst_event_new_segment (&src->segment); + + gst_event_set_seqnum (seg_event, src->priv->segment_seqnum); + src->priv->segment_seqnum = gst_util_seqnum_next (); + gst_base_idle_src_queue_object (src, (GstMiniObject *) seg_event); + GST_DEBUG_OBJECT (src, "Queing segment event %" GST_PTR_FORMAT, seg_event); + + src->priv->segment_pending = FALSE; + } +} + +static void +gst_base_idle_src_add_stream_start (GstBaseIdleSrc * src) +{ + gchar *stream_id; + GstEvent *event; + + stream_id = + gst_pad_create_stream_id (src->srcpad, GST_ELEMENT_CAST (src), NULL); + + GST_DEBUG_OBJECT (src, "Pushing STREAM_START"); + event = gst_event_new_stream_start (stream_id); + gst_event_set_group_id (event, gst_util_group_id_next ()); + + gst_base_idle_src_queue_object (src, (GstMiniObject *) event); + g_free (stream_id); +} + +static gboolean +gst_base_idle_src_start (GstBaseIdleSrc * src) +{ + GstBaseIdleSrcClass *bclass; + gboolean result; + + GST_DEBUG_OBJECT (src, "Starting"); + + GST_OBJECT_LOCK (src); + + gst_segment_init (&src->segment, src->segment.format); + GST_OBJECT_UNLOCK (src); + + src->running = TRUE; + src->priv->segment_pending = TRUE; + src->priv->segment_seqnum = gst_util_seqnum_next (); + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + if (bclass->start) + result = bclass->start (src); + else + result = TRUE; + + if (!result) + goto could_not_start; + + gst_base_idle_src_add_stream_start (src); + gst_base_idle_src_start_task (src, TRUE); + + return result; + +could_not_start: + { + GST_DEBUG_OBJECT (src, "could not start"); + /* subclass is supposed to post a message but we post one as a fallback + * just in case. We don't have to call _stop. */ + GST_ELEMENT_ERROR (src, CORE, STATE_CHANGE, (NULL), ("Failed to start")); + return FALSE; + } +} + +static gboolean +gst_base_idle_src_stop (GstBaseIdleSrc * src) +{ + GstBaseIdleSrcClass *bclass; + GstMiniObject *obj; + gboolean result = TRUE; + + GST_DEBUG_OBJECT (src, "stopping source"); + + src->running = FALSE; + if (src->priv->thread) { + g_thread_join (src->priv->thread); + src->priv->thread = NULL; + } + + /* clean up any leftovers on the queue */ + while ((obj = g_queue_pop_head (src->priv->obj_queue))) { + gst_mini_object_unref (obj); + } + + bclass = GST_BASE_IDLE_SRC_GET_CLASS (src); + if (bclass->stop) + result = bclass->stop (src); + + gst_base_idle_src_set_allocation (src, NULL, NULL, NULL); + + return result; +} + +static gboolean +gst_base_idle_src_activate_push (GstPad * pad, GstObject * parent, + gboolean active) +{ + GstBaseIdleSrc *basesrc; + + basesrc = GST_BASE_IDLE_SRC (parent); + + /* prepare subclass first */ + if (active) { + GST_DEBUG_OBJECT (basesrc, "Activating in push mode"); + + if (G_UNLIKELY (!gst_base_idle_src_start (basesrc))) + goto error_start; + } else { + GST_DEBUG_OBJECT (basesrc, "Deactivating in push mode"); + /* now we can stop the source */ + if (G_UNLIKELY (!gst_base_idle_src_stop (basesrc))) + goto error_stop; + } + return TRUE; + +error_start: + { + GST_WARNING_OBJECT (basesrc, "Failed to start in push mode"); + return FALSE; + } +error_stop: + { + GST_DEBUG_OBJECT (basesrc, "Failed to stop in push mode"); + return FALSE; + } +} + +static gboolean +gst_base_idle_src_activate_mode (GstPad * pad, GstObject * parent, + GstPadMode mode, gboolean active) +{ + gboolean res; + + GST_DEBUG_OBJECT (pad, "activating in mode %d", mode); + + switch (mode) { + case GST_PAD_MODE_PULL: + g_assert_not_reached (); + break; + case GST_PAD_MODE_PUSH: + res = gst_base_idle_src_activate_push (pad, parent, active); + break; + default: + GST_LOG_OBJECT (pad, "unknown activation mode %d", mode); + res = FALSE; + break; + } + return res; +} + + +/** + * gst_base_idle_src_submit_buffer: + * @src: a #GstBaseIdleSrc + * @buffer: (transfer full): a #GstBuffer + * + * Subclasses can call this to submit a buffer to be pushed out later. + * + * Since: 1.22 + */ +void +gst_base_idle_src_submit_buffer (GstBaseIdleSrc * src, GstBuffer * buffer) +{ + g_return_if_fail (GST_IS_BASE_IDLE_SRC (src)); + g_return_if_fail (GST_IS_BUFFER (buffer)); + + if (!src->running) { + GST_ERROR_OBJECT (src, "Sending buffer to stopped src is not valid"); + gst_buffer_unref (buffer); + return; + } + + gst_base_idle_src_check_pending_segment (src); + + /* we need it to be writable later in get_range() where we use get_writable */ + gst_base_idle_src_queue_object (src, (GstMiniObject *) buffer); + + gst_base_idle_src_start_task (src, FALSE); +} + +/** + * gst_base_idle_src_submit_buffer_list: + * @src: a #GstBaseIdleSrc + * @buffer_list: (transfer full): a #GstBufferList + * + * Subclasses can call this to submit a buffer list to be pushed out later. + * + * Since: 1.22 + */ +void +gst_base_idle_src_submit_buffer_list (GstBaseIdleSrc * src, + GstBufferList * buffer_list) +{ + g_return_if_fail (GST_IS_BASE_IDLE_SRC (src)); + g_return_if_fail (GST_IS_BUFFER_LIST (buffer_list)); + + if (!src->running) { + GST_ERROR_OBJECT (src, "Sending bufferlist to stopped src is not valid"); + gst_buffer_list_unref (buffer_list); + return; + } + + gst_base_idle_src_check_pending_segment (src); + + /* we need it to be writable later in get_range() where we use get_writable */ + gst_base_idle_src_queue_object (src, (GstMiniObject *) buffer_list); + + GST_LOG_OBJECT (src, "%u buffers submitted in buffer list", + gst_buffer_list_length (buffer_list)); + + gst_base_idle_src_start_task (src, FALSE); +} + + +/** + * gst_base_idle_src_alloc_buffer: + * @src: a #GstBaseIdleSrc + * @size: a gsize with the size of the buffer + * @buffer: (transfer full): a #GstBuffer + * + * Subclasses can call this to alloc a buffer. + * + * Since: 1.22 + */ +GstFlowReturn +gst_base_idle_src_alloc_buffer (GstBaseIdleSrc * src, + gsize size, GstBuffer ** buffer) +{ + GstFlowReturn ret; + GstBaseIdleSrcPrivate *priv = src->priv; + GstBufferPool *pool = NULL; + GstAllocator *allocator = NULL; + GstAllocationParams params; + + GST_OBJECT_LOCK (src); + if (priv->pool) { + pool = gst_object_ref (priv->pool); + } else if (priv->allocator) { + allocator = gst_object_ref (priv->allocator); + } + params = priv->params; + GST_OBJECT_UNLOCK (src); + + if (pool) { + ret = gst_buffer_pool_acquire_buffer (pool, buffer, NULL); + } else if (size != -1) { + *buffer = gst_buffer_new_allocate (allocator, size, ¶ms); + if (G_UNLIKELY (*buffer == NULL)) + goto alloc_failed; + + ret = GST_FLOW_OK; + } else { + GST_WARNING_OBJECT (src, "Not trying to alloc %u bytes. Blocksize not set?", + size); + goto alloc_failed; + } + +done: + if (pool) + gst_object_unref (pool); + if (allocator) + gst_object_unref (allocator); + + return ret; + + /* ERRORS */ +alloc_failed: + { + GST_ERROR_OBJECT (src, "Failed to allocate %u bytes", size); + ret = GST_FLOW_ERROR; + goto done; + } +} + +static void +gst_base_idle_src_finalize (GObject * object) +{ + GstBaseIdleSrc *src; + src = GST_BASE_IDLE_SRC (object); + + g_queue_free (src->priv->obj_queue); + /* FIXME: empty this queue potentially... */ + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_base_idle_src_class_init (GstBaseIdleSrcClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + + gobject_class = G_OBJECT_CLASS (klass); + gstelement_class = GST_ELEMENT_CLASS (klass); + + if (private_offset != 0) + g_type_class_adjust_private_offset (klass, &private_offset); + + GST_DEBUG_CATEGORY_INIT (gst_base_idle_src_debug, "idlesrc", 0, + "idlesrc element"); + + parent_class = g_type_class_peek_parent (klass); + + gobject_class->finalize = gst_base_idle_src_finalize; + gobject_class->set_property = gst_base_idle_src_set_property; + gobject_class->get_property = gst_base_idle_src_get_property; + + g_object_class_install_property (gobject_class, PROP_DO_TIMESTAMP, + g_param_spec_boolean ("do-timestamp", "Do timestamp", + "Apply current stream time to buffers", DEFAULT_DO_TIMESTAMP, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + gstelement_class->send_event = + GST_DEBUG_FUNCPTR (gst_base_idle_src_send_event); + + klass->get_caps = GST_DEBUG_FUNCPTR (gst_base_idle_src_default_get_caps); + klass->negotiate = GST_DEBUG_FUNCPTR (gst_base_idle_src_default_negotiate); + klass->fixate = GST_DEBUG_FUNCPTR (gst_base_idle_src_default_fixate); + klass->query = GST_DEBUG_FUNCPTR (gst_base_idle_src_default_query); + klass->event = GST_DEBUG_FUNCPTR (gst_base_idle_src_default_event); + klass->decide_allocation = + GST_DEBUG_FUNCPTR (gst_base_idle_src_decide_allocation_default); + + /* Registering debug symbols for function pointers */ + + GST_DEBUG_REGISTER_FUNCPTR (gst_base_idle_src_event); + GST_DEBUG_REGISTER_FUNCPTR (gst_base_idle_src_query); + GST_DEBUG_REGISTER_FUNCPTR (gst_base_idle_src_fixate); +} + +static void +gst_base_idle_src_init (GstBaseIdleSrc * src, gpointer g_class) +{ + GstPad *pad; + GstPadTemplate *pad_template; + + src->priv = gst_base_idle_src_get_instance_private (src); + + pad_template = + gst_element_class_get_pad_template (GST_ELEMENT_CLASS (g_class), "src"); + g_return_if_fail (pad_template != NULL); + + GST_DEBUG_OBJECT (src, "creating src pad"); + pad = gst_pad_new_from_template (pad_template, "src"); + + GST_DEBUG_OBJECT (src, "setting functions on src pad"); + gst_pad_set_activatemode_function (pad, gst_base_idle_src_activate_mode); + gst_pad_set_event_function (pad, gst_base_idle_src_event); + gst_pad_set_query_function (pad, gst_base_idle_src_query); + + /* hold pointer to pad */ + src->srcpad = pad; + GST_DEBUG_OBJECT (src, "adding src pad"); + gst_element_add_pad (GST_ELEMENT (src), pad); + + /* we operate in BYTES by default */ + gst_base_idle_src_set_format (src, GST_FORMAT_BYTES); + src->priv->do_timestamp = DEFAULT_DO_TIMESTAMP; + + GST_OBJECT_FLAG_SET (src, GST_ELEMENT_FLAG_SOURCE); + + src->priv->obj_queue = g_queue_new (); + + GST_DEBUG_OBJECT (src, "init done"); +} + +GType +gst_base_idle_src_get_type (void) +{ + static gsize base_idle_src_type = 0; + + if (g_once_init_enter (&base_idle_src_type)) { + GType _type; + static const GTypeInfo base_idle_src_info = { + sizeof (GstBaseIdleSrcClass), + NULL, + NULL, + (GClassInitFunc) gst_base_idle_src_class_init, + NULL, + NULL, + sizeof (GstBaseIdleSrc), + 0, + (GInstanceInitFunc) gst_base_idle_src_init, + }; + + _type = g_type_register_static (GST_TYPE_ELEMENT, + "GstBaseIdleSrc", &base_idle_src_info, G_TYPE_FLAG_ABSTRACT); + + private_offset = + g_type_add_instance_private (_type, sizeof (GstBaseIdleSrcPrivate)); + + g_once_init_leave (&base_idle_src_type, _type); + } + return base_idle_src_type; +} diff --git a/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.h b/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.h new file mode 100644 index 00000000000..cf9fea5660a --- /dev/null +++ b/subprojects/gstreamer/libs/gst/base/gstbaseidlesrc.h @@ -0,0 +1,220 @@ +/* GStreamer + * Copyright (C) 1999,2000 Erik Walthinsen + * 2000 Wim Taymans + * 2005 Wim Taymans + * + * gstbaseidlesrc.h: + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_BASE_IDLE_SRC_H__ +#define __GST_BASE_IDLE_SRC_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_BASE_IDLE_SRC (gst_base_idle_src_get_type()) +#define GST_BASE_IDLE_SRC(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_BASE_IDLE_SRC,GstBaseIdleSrc)) +#define GST_BASE_IDLE_SRC_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_BASE_IDLE_SRC,GstBaseIdleSrcClass)) +#define GST_BASE_IDLE_SRC_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_BASE_IDLE_SRC, GstBaseIdleSrcClass)) +#define GST_IS_BASE_IDLE_SRC(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_BASE_IDLE_SRC)) +#define GST_IS_BASE_IDLE_SRC_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_BASE_IDLE_SRC)) +#define GST_BASE_IDLE_SRC_CAST(obj) ((GstBaseIdleSrc *)(obj)) + +typedef struct _GstBaseIdleSrc GstBaseIdleSrc; +typedef struct _GstBaseIdleSrcClass GstBaseIdleSrcClass; +typedef struct _GstBaseIdleSrcPrivate GstBaseIdleSrcPrivate; + +/** + * GST_BASE_IDLE_SRC_PAD: + * @obj: base source instance + * + * Gives the pointer to the #GstPad object of the element. + */ +#define GST_BASE_IDLE_SRC_PAD(obj) (GST_BASE_IDLE_SRC_CAST (obj)->srcpad) + +/** + * GstBaseIdleSrc: + * + * The opaque #GstBaseIdleSrc data structure. + */ +struct _GstBaseIdleSrc { + GstElement element; + + /*< protected >*/ + GstPad *srcpad; + + gboolean is_live; + + /* MT-protected (with STREAM_LOCK *and* OBJECT_LOCK) */ + GstSegment segment; + /* MT-protected (with STREAM_LOCK) */ + gboolean need_newsegment; + + gint num_buffers; + gint num_buffers_left; + + gboolean running; + + GstBaseIdleSrcPrivate *priv; + + /*< private >*/ + gpointer _gst_reserved[GST_PADDING_LARGE]; +}; + +/** + * GstBaseIdleSrcClass: + * @parent_class: Element parent class + * @get_caps: Called to get the caps to report + * @negotiate: Negotiated the caps with the peer. + * @fixate: Called during negotiation if caps need fixating. Implement instead of + * setting a fixate function on the source pad. + * @set_caps: Notify subclass of changed output caps + * @decide_allocation: configure the allocation query + * @start: Start processing. Subclasses should open resources and prepare + * to produce data. Implementation should call gst_base_idle_src_start_complete() + * when the operation completes, either from the current thread or any other + * thread that finishes the start operation asynchronously. + * @stop: Stop processing. Subclasses should use this to close resources. + * @get_size: Return the total size of the resource, in the format set by + * gst_base_idle_src_set_format(). + * @query: Handle a requested query. + * @event: Override this to implement custom event handling. + * @alloc: Ask the subclass to allocate a buffer with for offset and size. The + * default implementation will create a new buffer from the negotiated allocator. + * @fill: Ask the subclass to fill the buffer with data for offset and size. The + * passed buffer is guaranteed to hold the requested amount of bytes. + * + * Subclasses can override any of the available virtual methods or not, as + * needed. At the minimum, the @create method should be overridden to produce + * buffers. + */ +struct _GstBaseIdleSrcClass { + GstElementClass parent_class; + + /*< public >*/ + /* virtual methods for subclasses */ + + /** + * GstBaseIdleSrcClass::get_caps: + * @filter: (in) (nullable): + * + * Called to get the caps to report. + */ + GstCaps* (*get_caps) (GstBaseIdleSrc *src, GstCaps *filter); + /* decide on caps */ + gboolean (*negotiate) (GstBaseIdleSrc *src); + /* called if, in negotiation, caps need fixating */ + GstCaps * (*fixate) (GstBaseIdleSrc *src, GstCaps *caps); + /* notify the subclass of new caps */ + gboolean (*set_caps) (GstBaseIdleSrc *src, GstCaps *caps); + + /* setup allocation query */ + gboolean (*decide_allocation) (GstBaseIdleSrc *src, GstQuery *query); + + /* start and stop processing, ideal for opening/closing the resource */ + gboolean (*start) (GstBaseIdleSrc *src); + gboolean (*stop) (GstBaseIdleSrc *src); + + /** + * GstBaseIdleSrcClass::get_size: + * @size: (out): + * + * Get the total size of the resource in the format set by + * gst_base_idle_src_set_format(). + * + * Returns: %TRUE if the size is available and has been set. + */ + gboolean (*get_size) (GstBaseIdleSrc *src, guint64 *size); + + /* notify subclasses of a query */ + gboolean (*query) (GstBaseIdleSrc *src, GstQuery *query); + + /* notify subclasses of an event */ + gboolean (*event) (GstBaseIdleSrc *src, GstEvent *event); + + + /*< private >*/ + gpointer _gst_reserved[GST_PADDING_LARGE]; +}; + +GST_BASE_API +GType gst_base_idle_src_get_type (void); + +GST_BASE_API +void gst_base_idle_src_set_live (GstBaseIdleSrc *src, gboolean live); + +GST_BASE_API +gboolean gst_base_idle_src_is_live (GstBaseIdleSrc *src); + +GST_BASE_API +void gst_base_idle_src_set_format (GstBaseIdleSrc *src, GstFormat format); + +GST_BASE_API +void gst_base_idle_src_set_automatic_eos (GstBaseIdleSrc * src, gboolean automatic_eos); + + +GST_BASE_API +gboolean gst_base_idle_src_negotiate (GstBaseIdleSrc *src); + + +GST_BASE_API +gboolean gst_base_idle_src_query_latency (GstBaseIdleSrc *src, gboolean * live, + GstClockTime * min_latency, + GstClockTime * max_latency); + +GST_BASE_API +void gst_base_idle_src_set_do_timestamp (GstBaseIdleSrc *src, gboolean timestamp); + +GST_BASE_API +gboolean gst_base_idle_src_get_do_timestamp (GstBaseIdleSrc *src); + +GST_BASE_API +gboolean gst_base_idle_src_new_segment (GstBaseIdleSrc *src, + const GstSegment * segment); + +GST_BASE_API +gboolean gst_base_idle_src_set_caps (GstBaseIdleSrc *src, GstCaps *caps); + +GST_BASE_API +GstBufferPool * gst_base_idle_src_get_buffer_pool (GstBaseIdleSrc *src); + +GST_BASE_API +void gst_base_idle_src_get_allocator (GstBaseIdleSrc *src, + GstAllocator **allocator, + GstAllocationParams *params); + +GST_BASE_API +void gst_base_idle_src_submit_buffer (GstBaseIdleSrc * src, + GstBuffer * buffer); + +GST_BASE_API +void gst_base_idle_src_submit_buffer_list (GstBaseIdleSrc * src, + GstBufferList * buffer_list); + +GST_BASE_API +GstFlowReturn gst_base_idle_src_alloc_buffer (GstBaseIdleSrc * src, + gsize size, + GstBuffer ** buffer); + +G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstBaseIdleSrc, gst_object_unref) + +G_END_DECLS + +#endif /* __GST_BASE_IDLE_SRC_H__ */ diff --git a/subprojects/gstreamer/libs/gst/base/gstbasetransform.c b/subprojects/gstreamer/libs/gst/base/gstbasetransform.c index 3f38c094b20..52b1ed95ff0 100644 --- a/subprojects/gstreamer/libs/gst/base/gstbasetransform.c +++ b/subprojects/gstreamer/libs/gst/base/gstbasetransform.c @@ -163,18 +163,26 @@ enum PROP_QOS }; +#define PRIV_LOCK(trans) g_rec_mutex_lock (&(trans)->priv->lock) +#define PRIV_UNLOCK(trans) g_rec_mutex_unlock (&(trans)->priv->lock) + struct _GstBaseTransformPrivate { /* Set by sub-class */ gboolean passthrough; gboolean always_in_place; + GRecMutex lock; + GstCaps *cache_caps1; gsize cache_caps1_size; GstCaps *cache_caps2; gsize cache_caps2_size; gboolean have_same_caps; + GstCaps *cache_incaps; + GstCaps *cache_outcaps; + gboolean negotiated; /* QoS *//* with LOCK */ @@ -314,6 +322,10 @@ gst_base_transform_default_transform_meta (GstBaseTransform * trans, static void gst_base_transform_finalize (GObject * object) { + GstBaseTransform *trans = GST_BASE_TRANSFORM_CAST (object); + + g_rec_mutex_clear (&trans->priv->lock); + G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -381,6 +393,7 @@ gst_base_transform_init (GstBaseTransform * trans, GST_DEBUG ("gst_base_transform_init"); priv = trans->priv = gst_base_transform_get_instance_private (trans); + g_rec_mutex_init (&priv->lock); pad_template = gst_element_class_get_pad_template (GST_ELEMENT_CLASS (bclass), "sink"); @@ -753,7 +766,7 @@ gst_base_transform_set_allocation (GstBaseTransform * trans, GstQuery *oldquery; GstBaseTransformPrivate *priv = trans->priv; - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); oldpool = priv->pool; priv->pool = pool; priv->pool_active = FALSE; @@ -768,7 +781,7 @@ gst_base_transform_set_allocation (GstBaseTransform * trans, priv->params = *params; else gst_allocation_params_init (&priv->params); - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); if (oldpool) { GST_DEBUG_OBJECT (trans, "deactivating old pool %p", oldpool); @@ -1019,6 +1032,9 @@ gst_base_transform_configure_caps (GstBaseTransform * trans, GstCaps * in, gst_caps_replace (&priv->cache_caps1, NULL); gst_caps_replace (&priv->cache_caps2, NULL); + gst_caps_replace (&priv->cache_incaps, in); + gst_caps_replace (&priv->cache_outcaps, out); + /* figure out same caps state */ priv->have_same_caps = gst_caps_is_equal (in, out); GST_DEBUG_OBJECT (trans, "have_same_caps: %d", priv->have_same_caps); @@ -1359,9 +1375,9 @@ gst_base_transform_setcaps (GstBaseTransform * trans, GstPad * pad, if (prev_outcaps) gst_caps_unref (prev_outcaps); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); priv->negotiated = ret; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); return ret; @@ -1492,13 +1508,21 @@ gst_base_transform_default_query (GstBaseTransform * trans, GstPad *pad, *otherpad; GstBaseTransformClass *klass; GstBaseTransformPrivate *priv = trans->priv; + GstElement *el = GST_ELEMENT_CAST (trans); + GST_OBJECT_LOCK (el); if (direction == GST_PAD_SRC) { - pad = trans->srcpad; - otherpad = trans->sinkpad; + pad = el->srcpads ? gst_object_ref (el->srcpads->data) : NULL; + otherpad = el->sinkpads ? gst_object_ref (el->sinkpads->data) : NULL; } else { - pad = trans->sinkpad; - otherpad = trans->srcpad; + pad = el->sinkpads ? gst_object_ref (el->sinkpads->data) : NULL; + otherpad = el->srcpads ? gst_object_ref (el->srcpads->data) : NULL; + } + GST_OBJECT_UNLOCK (el); + + if (!pad || !otherpad) { + GST_WARNING_OBJECT (trans, "Our pads are missing!"); + goto done; } klass = GST_BASE_TRANSFORM_GET_CLASS (trans); @@ -1516,17 +1540,17 @@ gst_base_transform_default_query (GstBaseTransform * trans, if (G_UNLIKELY (!ret)) goto done; - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); if (!priv->negotiated && !priv->passthrough && (klass->set_caps != NULL)) { GST_DEBUG_OBJECT (trans, "not negotiated yet but need negotiation, can't answer ALLOCATION query"); - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); goto done; } decide_query = trans->priv->query; trans->priv->query = NULL; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); GST_DEBUG_OBJECT (trans, "calling propose allocation with query %" GST_PTR_FORMAT, @@ -1539,14 +1563,14 @@ gst_base_transform_default_query (GstBaseTransform * trans, ret = FALSE; if (decide_query) { - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); if (trans->priv->query == NULL) trans->priv->query = decide_query; else gst_query_unref (decide_query); - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); } GST_DEBUG_OBJECT (trans, "ALLOCATION ret %d, %" GST_PTR_FORMAT, ret, @@ -1611,6 +1635,10 @@ gst_base_transform_default_query (GstBaseTransform * trans, } done: + if (pad) + gst_object_unref (pad); + if (otherpad) + gst_object_unref (otherpad); return ret; } @@ -1686,8 +1714,8 @@ default_prepare_output_buffer (GstBaseTransform * trans, } /* else use the transform function to get the size */ - incaps = gst_pad_get_current_caps (trans->sinkpad); - outcaps = gst_pad_get_current_caps (trans->srcpad); + incaps = priv->cache_incaps; + outcaps = priv->cache_outcaps; /* srcpad might be flushing already if we're being shut down */ if (outcaps == NULL) @@ -1699,9 +1727,6 @@ default_prepare_output_buffer (GstBaseTransform * trans, res = gst_base_transform_transform_size (trans, GST_PAD_SINK, incaps, insize, outcaps, &outsize); - gst_caps_unref (incaps); - gst_caps_unref (outcaps); - if (!res) goto unknown_size; @@ -1924,14 +1949,14 @@ gst_base_transform_sink_eventfunc (GstBaseTransform * trans, GstEvent * event) case GST_EVENT_FLUSH_START: break; case GST_EVENT_FLUSH_STOP: - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); /* reset QoS parameters */ priv->proportion = 1.0; priv->earliest_time = -1; priv->discont = FALSE; priv->processed = 0; priv->dropped = 0; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); /* we need new segment info after the flush. */ trans->have_segment = FALSE; gst_segment_init (&trans->segment, GST_FORMAT_UNDEFINED); @@ -2083,13 +2108,14 @@ default_submit_input_buffer (GstBaseTransform * trans, gboolean is_discont, /* lock for getting the QoS parameters that are set (in a different thread) * with the QOS events */ - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); earliest_time = priv->earliest_time; proportion = priv->proportion; /* check for QoS, don't perform conversion for buffers * that are known to be late. */ - need_skip = earliest_time != -1 && running_time <= earliest_time; - GST_OBJECT_UNLOCK (trans); + need_skip = priv->qos_enabled && + earliest_time != -1 && running_time <= earliest_time; + PRIV_UNLOCK (trans); if (need_skip) { GstMessage *qos_msg; @@ -2324,6 +2350,8 @@ gst_base_transform_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) GstClockTime timestamp, duration; GstBuffer *outbuf = NULL; + PRIV_LOCK (trans); + timestamp = GST_BUFFER_TIMESTAMP (buffer); duration = GST_BUFFER_DURATION (buffer); @@ -2345,7 +2373,9 @@ gst_base_transform_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) } /* Takes ownership of input buffer */ + PRIV_UNLOCK (trans); ret = klass->submit_input_buffer (trans, priv->discont, buffer); + PRIV_LOCK (trans); if (ret != GST_FLOW_OK) goto done; @@ -2354,6 +2384,8 @@ gst_base_transform_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) ret = klass->generate_output (trans, &outbuf); + PRIV_UNLOCK (trans); + /* outbuf can be NULL, this means a dropped buffer, if we have a buffer but * GST_BASE_TRANSFORM_FLOW_DROPPED we will not push either. */ if (outbuf != NULL) { @@ -2394,9 +2426,12 @@ gst_base_transform_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) gst_buffer_unref (outbuf); } } + PRIV_LOCK (trans); } while (ret == GST_FLOW_OK && outbuf != NULL); done: + PRIV_UNLOCK (trans); + /* convert internal flow to OK and mark discont for the next buffer. */ if (ret == GST_BASE_TRANSFORM_FLOW_DROPPED) { GST_DEBUG_OBJECT (trans, "dropped a buffer, marking DISCONT"); @@ -2462,7 +2497,7 @@ gst_base_transform_activate (GstBaseTransform * trans, gboolean active) incaps = gst_pad_get_current_caps (trans->sinkpad); outcaps = gst_pad_get_current_caps (trans->srcpad); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); if (incaps && outcaps) priv->have_same_caps = gst_caps_is_equal (incaps, outcaps) || priv->passthrough; @@ -2478,7 +2513,7 @@ gst_base_transform_activate (GstBaseTransform * trans, gboolean active) priv->discont = FALSE; priv->processed = 0; priv->dropped = 0; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); if (incaps) gst_caps_unref (incaps); @@ -2499,6 +2534,9 @@ gst_base_transform_activate (GstBaseTransform * trans, gboolean active) gst_caps_replace (&priv->cache_caps1, NULL); gst_caps_replace (&priv->cache_caps2, NULL); + gst_caps_replace (&priv->cache_incaps, NULL); + gst_caps_replace (&priv->cache_outcaps, NULL); + /* Make sure any left over buffer is freed */ gst_buffer_replace (&trans->queued_buf, NULL); @@ -2590,7 +2628,7 @@ gst_base_transform_set_passthrough (GstBaseTransform * trans, bclass = GST_BASE_TRANSFORM_GET_CLASS (trans); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); if (!passthrough) { if (bclass->transform_ip || bclass->transform || (bclass->generate_output && bclass->generate_output != default_generate_output)) @@ -2600,7 +2638,7 @@ gst_base_transform_set_passthrough (GstBaseTransform * trans, } GST_DEBUG_OBJECT (trans, "set passthrough %d", trans->priv->passthrough); - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); } /** @@ -2620,9 +2658,9 @@ gst_base_transform_is_passthrough (GstBaseTransform * trans) g_return_val_if_fail (GST_IS_BASE_TRANSFORM (trans), FALSE); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); result = trans->priv->passthrough; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); return result; } @@ -2650,7 +2688,7 @@ gst_base_transform_set_in_place (GstBaseTransform * trans, gboolean in_place) bclass = GST_BASE_TRANSFORM_GET_CLASS (trans); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); if (in_place) { if (bclass->transform_ip) { @@ -2664,7 +2702,7 @@ gst_base_transform_set_in_place (GstBaseTransform * trans, gboolean in_place) } } - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); } /** @@ -2684,9 +2722,9 @@ gst_base_transform_is_in_place (GstBaseTransform * trans) g_return_val_if_fail (GST_IS_BASE_TRANSFORM (trans), FALSE); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); result = trans->priv->always_in_place; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); return result; } @@ -2716,10 +2754,10 @@ gst_base_transform_update_qos (GstBaseTransform * trans, "qos: proportion: %lf, diff %" G_GINT64_FORMAT ", timestamp %" GST_TIME_FORMAT, proportion, diff, GST_TIME_ARGS (timestamp)); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); trans->priv->proportion = proportion; trans->priv->earliest_time = timestamp + diff; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); } /** @@ -2738,9 +2776,9 @@ gst_base_transform_set_qos_enabled (GstBaseTransform * trans, gboolean enabled) GST_CAT_DEBUG_OBJECT (GST_CAT_QOS, trans, "enabled: %d", enabled); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); trans->priv->qos_enabled = enabled; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); } /** @@ -2760,9 +2798,9 @@ gst_base_transform_is_qos_enabled (GstBaseTransform * trans) g_return_val_if_fail (GST_IS_BASE_TRANSFORM (trans), FALSE); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); result = trans->priv->qos_enabled; - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); return result; } @@ -2786,10 +2824,10 @@ gst_base_transform_set_gap_aware (GstBaseTransform * trans, gboolean gap_aware) { g_return_if_fail (GST_IS_BASE_TRANSFORM (trans)); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); trans->priv->gap_aware = gap_aware; GST_DEBUG_OBJECT (trans, "set gap aware %d", trans->priv->gap_aware); - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); } /** @@ -2817,10 +2855,10 @@ gst_base_transform_set_prefer_passthrough (GstBaseTransform * trans, { g_return_if_fail (GST_IS_BASE_TRANSFORM (trans)); - GST_OBJECT_LOCK (trans); + PRIV_LOCK (trans); trans->priv->prefer_passthrough = prefer_passthrough; GST_DEBUG_OBJECT (trans, "prefer passthrough %d", prefer_passthrough); - GST_OBJECT_UNLOCK (trans); + PRIV_UNLOCK (trans); } /** diff --git a/subprojects/gstreamer/libs/gst/base/gstbytewriter.h b/subprojects/gstreamer/libs/gst/base/gstbytewriter.h index 365c7742d73..f2ba7530ba5 100644 --- a/subprojects/gstreamer/libs/gst/base/gstbytewriter.h +++ b/subprojects/gstreamer/libs/gst/base/gstbytewriter.h @@ -374,7 +374,7 @@ gst_byte_writer_put_buffer_unchecked (GstByteWriter * writer, GstBuffer * buffer gst_buffer_extract (buffer, offset, (guint8 *) & writer->parent.data[writer->parent.byte], size); - writer->parent.byte += size; + writer->parent.byte += (guint) size; writer->parent.size = MAX (writer->parent.size, writer->parent.byte); } @@ -394,7 +394,7 @@ _gst_byte_writer_put_buffer_inline (GstByteWriter * writer, GstBuffer * buffer, size -= offset; } - if (G_UNLIKELY (!_gst_byte_writer_ensure_free_space_inline (writer, size))) + if (G_UNLIKELY (!_gst_byte_writer_ensure_free_space_inline (writer, (guint) size))) return FALSE; gst_byte_writer_put_buffer_unchecked (writer, buffer, offset, size); diff --git a/subprojects/gstreamer/libs/gst/base/gstpriqueue.c b/subprojects/gstreamer/libs/gst/base/gstpriqueue.c new file mode 100644 index 00000000000..adc12e02216 --- /dev/null +++ b/subprojects/gstreamer/libs/gst/base/gstpriqueue.c @@ -0,0 +1,900 @@ +/* GStreamer + * Copyright (C) 2016 Pexip AS + * Erlend Graff + * + * gstpriqueue.c: binomial heap implementation of a priority queue + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include "gstpriqueue.h" + +#include +#include +#include +#include + +/** + * SECTION:gstpriqueue + * @title: Priority Queues + * @short_description: collections optimized for dequeuing elements in priority + * order + * + * #GstPriQueue provides the API of a priority queue, where the only supported + * lookup operation is to find/dequeue the element with the highest priority, + * which is defined as the smallest value according to the + * supplied comparison function. + * + * The #GstPriQueue is implemented internally as a binomial heap. This makes it + * possible to find or remove the smallest element with an upper time bound of + * `O(log n)`. Insertions have a worst-case time of `O(log n)`, but an amortized + * asymptotic bound of `O(1)` over `n` consecutive insertions. Deletions are + * supported in `O(log n)`, but deleting an arbitrary element (other than the + * smallest element) is much more expensive than an insertion (up to a constant + * factor of approximately 5 in the worst case). An update operation + * (reinsertion of an element after its value has changed) is also implemented + * in `O(log n)`, and is faster than a corresponding delete + (re-)insert. + * Finally, the binomial heap structure also supports merging two priority + * queues, in `O(log n)` time. + * + * The #GstPriQueue does not handle memory allocation of its elements. Instead, + * the responsibility for allocating and freeing a #GstPriQueueElem, whenever + * it is inserted into or removed from the #GstPriQueue, is given to the caller. + * Typically, the caller will embed the #GstPriQueueElem structure inside a + * containing "parent" structure that will represent the value of the element. + * For example: + * |[ + * typedef struct + * { + * ... + * GstPriQueueElem pq_elem; + * ... + * } SomeValue; + * + * ... + * + * SomeValue *val = create_new_some_value (); + * + * gst_pri_queue_insert (pq, &val->pq_elem); + * ]| + * The GST_CONTAINER_OF() macro defined in gst/gstmacros.h + * can then be used to retrieve the containing `SomeValue` of a + * #GstPriQueueElem: + * |[ + * GstPriQueueElem *elem; + * SomeValue *val; + * + * elem = gst_pri_queue_pop_min (pq); + * if (elem) { + * val = GST_CONTAINER_OF (elem, SomeValue, pq_elem); + * ... + * } + * ]| + * The comparison function will also typically make use of GST_CONTAINER_OF(): + * |[ + * static gint + * compare_some_value_pq_elems (const GstPriQueueElem * a, + * const GstPriQueueElem * b, gpointer user_data) + * { + * SomeValue *val_a = GST_CONTAINER_OF (a, SomeValue, pq_elem); + * SomeValue *val_b = GST_CONTAINER_OF (b, SomeValue, pq_elem); + * + * return compare_some_value (val_a, val_b, user_data); + * } + * + * ... + * + * GstPriQueue *pq; + * + * pq = gst_pri_queue_create (compare_some_value_pq_elems, some_user_data); + * ]| + */ + +#define NODE2ELEM(nodeptr) (GST_CONTAINER_OF ((nodeptr), GstPriQueueElem, node)) +#define ELEM2NODE(elemptr) (&(elemptr)->node) + +typedef struct _GstPriQueueNode GstPriQueueNode; + +/** + * GstPriQueue: (skip) + * + * The #GstPriQueue struct is an opaque data type representing a priority + * queue. + */ +struct _GstPriQueue +{ + GstPriQueueNode *head; + GstPriQueueCompareFunc cmp_func; + gpointer user_data; + gsize size; +}; + +/** + * GstPriQueueElem: (skip) + * + * The #GstPriQueueElem struct represents an element in a #GstPriQueue. + * This struct must be treated as an opaque data type, but has a public + * definition to allow it to be embedded in another "parent" structure + * representing the value of an element. + */ + +/** + * GstPriQueueIter: (skip) + * + * The #GstPriQueueIter struct represents an iterator that can be used to + * iterate over all the elements in a #GstPriQueue. This struct must be treated + * as an opaque data type, but has a public definition to allow it to be + * allocated on the stack. + * + * A #GstPriQueueIter is initialized using gst_pri_queue_iter_init(), and + * gst_pri_queue_iter_next() is used to do the actual iteration. Note that + * iteration over elements in a #GstPriQueue is performed in an arbitrary order, + * and not in priority order. The first element of an + * iteration sequence is not even guaranteed to be the smallest element returned + * by gst_pri_queue_get_min() or gst_pri_queue_pop_min(), or any of the smallest + * elements in the #GstPriQueue if there are multiple such. + */ + +typedef struct +{ + GstPriQueueNode *parent; + GstPriQueueNode **list_pos; + GstPriQueueNode *children_head; + gint order; +} BinomTreePos; + +static inline gint +compare_nodes (const GstPriQueue * pq, GstPriQueueNode * a, GstPriQueueNode * b) +{ + return pq->cmp_func (NODE2ELEM (a), NODE2ELEM (b), pq->user_data); +} + +/* Remove @delnode from the list pointed to by @head. This may change what @head + * points to. + * + * Note: @head MUST be the actual head of the list! + */ +static inline GstPriQueueNode ** +list_remove_node (GstPriQueueNode ** head, GstPriQueueNode * delnode) +{ + GstPriQueueNode **pnext; + GstPriQueueNode *node; + + for (pnext = head; (node = *pnext) != delnode;) + pnext = &node->next; + + *pnext = node->next; + return pnext; +} + +/* Insert node at given position in list */ +static inline void +list_insert_node (GstPriQueueNode ** pnext, GstPriQueueNode * insnode) +{ + insnode->next = *pnext; + *pnext = insnode; +} + +static inline void +init_node (GstPriQueueNode * node) +{ + node->parent = NULL; + node->children_head = NULL; + node->next = NULL; + node->order = 0; +} + +/* Merge two binomial trees of the same order `n` into a binomial tree of order + * `(n + 1)`. + */ +static inline GstPriQueueNode * +merge_tree (const GstPriQueue * pq, GstPriQueueNode * a, GstPriQueueNode * b) +{ + GstPriQueueNode *new_root, *new_subtree; + + if (compare_nodes (pq, a, b) <= 0) { + new_root = a; + new_subtree = b; + } else { + new_root = b; + new_subtree = a; + } + + list_insert_node (&new_root->children_head, new_subtree); + new_subtree->parent = new_root; + new_root->order++; + + return new_root; +} + +/* Reverses list, and sets parent to %NULL. */ +static inline GstPriQueueNode * +subtree_list_to_heap_list (GstPriQueueNode * head) +{ + GstPriQueueNode *node, *new_head, *next; + + new_head = NULL; + node = head; + while (node) { + next = node->next; + node->next = new_head; + new_head = node; + node->parent = NULL; + node = next; + } + + return new_head; +} + +static inline GstPriQueueNode ** +get_containing_list (GstPriQueue * pq, GstPriQueueNode * node) +{ + return node->parent ? &node->parent->children_head : &pq->head; +} + +static inline GstPriQueueNode ** +remove_node_from_containing_list (GstPriQueue * pq, GstPriQueueNode * delnode) +{ + return list_remove_node (get_containing_list (pq, delnode), delnode); +} + +/* Add a single node to the given binomial heap list (list of root binomial + * trees). This will likely change the head of the binomial heap list. + */ +static inline void +binom_heap_list_add_node (GstPriQueue * pq, GstPriQueueNode ** head, + GstPriQueueNode * insnode) +{ + GstPriQueueNode *next; + + while ((next = *head) && insnode->order == next->order) { + (void) list_remove_node (head, next); + insnode = merge_tree (pq, next, insnode); + } + + list_insert_node (head, insnode); +} + +/* Merge the binomial heap list (list of root binomial trees) pointed to by + * @head_b (B-list) into @pq's binomial heap list (A-list). This will modify + * the A-list, and destroy the B-list. + */ +static void +binom_heap_union (GstPriQueue * pq, GstPriQueueNode * head_b) +{ + GstPriQueueNode **pnext_a; + GstPriQueueNode *next_a, *node; + + pnext_a = &pq->head; + while ((next_a = *pnext_a) && head_b) { + if (head_b->order > next_a->order) { + pnext_a = &next_a->next; + } else if (head_b->order < next_a->order) { + /* Remove head of B-list, and insert into A-list */ + node = head_b; + (void) list_remove_node (&head_b, node); + list_insert_node (pnext_a, node); + } else { + /* Remove node from A-list, and add to B-list */ + node = next_a; + (void) list_remove_node (pnext_a, node); + binom_heap_list_add_node (pq, &head_b, node); + } + } + + if (head_b) { + /* A-list exhausted, append rest of B-list to end */ + *pnext_a = head_b; + } +} + +static inline void +remove_tree_node (GstPriQueue * pq, GstPriQueueNode * delnode, + BinomTreePos * pos) +{ + pos->parent = delnode->parent; + pos->list_pos = remove_node_from_containing_list (pq, delnode); + pos->children_head = delnode->children_head; + pos->order = delnode->order; +} + +/* Note: this function does not set insnode->parent. */ +static inline void +insert_tree_node (BinomTreePos pos, GstPriQueueNode * insnode) +{ + GstPriQueueNode *child; + + for (child = pos.children_head; child; child = child->next) + child->parent = insnode; + + insnode->order = pos.order; + list_insert_node (pos.list_pos, insnode); + insnode->children_head = pos.children_head; +} + +static inline gboolean +should_decrease (GstPriQueue * pq, GstPriQueueNode * parent, + GstPriQueueNode * node, gboolean node_is_minus_inf) +{ + return parent && (node_is_minus_inf || compare_nodes (pq, node, parent) < 0); +} + +/* Try to move the given node upwards in its binomial tree until the min-heap + * invariant is satisfied. If @is_minus_if is %TRUE, the node will end up at + * the root of its binomial tree, otherwise its new position is determined by + * its value according to the comparison function. + * + * Returns %TRUE if @node was moved upwards in its binomial tree, else + * %FALSE. + */ +static inline gboolean +decrease_key (GstPriQueue * pq, GstPriQueueNode * node, gboolean is_minus_inf) +{ + BinomTreePos current_pos, parent_pos; + + if (!should_decrease (pq, node->parent, node, is_minus_inf)) + return FALSE; + + remove_tree_node (pq, node, ¤t_pos); + + do { + remove_tree_node (pq, current_pos.parent, &parent_pos); + insert_tree_node (current_pos, current_pos.parent); + + /* Check if current_pos.parent was inserted at head of its new containing + * list. If so, we must manually update parent_pos.children_head. + */ + if (parent_pos.children_head == current_pos.parent->next) + parent_pos.children_head = current_pos.parent; + + current_pos = parent_pos; + } while (should_decrease (pq, current_pos.parent, node, is_minus_inf)); + + insert_tree_node (current_pos, node); + node->parent = current_pos.parent; + + return TRUE; +} + +/* Update node's position in its own binomial subtree after its value might + * have changed. + */ +static inline void +increase_key (GstPriQueue * pq, GstPriQueueNode * node) +{ + GstPriQueueNode **list_pos; + GstPriQueueNode *parent, *head; + + parent = node->parent; + list_pos = remove_node_from_containing_list (pq, node); + + head = subtree_list_to_heap_list (node->children_head); + node->children_head = NULL; + node->order = 0; + binom_heap_list_add_node (pq, &head, node); + + list_insert_node (list_pos, head); + head->parent = parent; +} + +static inline void +remove_heap_root (GstPriQueue * pq, GstPriQueueNode * delnode) +{ + (void) list_remove_node (&pq->head, delnode); + binom_heap_union (pq, subtree_list_to_heap_list (delnode->children_head)); +} + +static inline GstPriQueueNode * +get_min_root (const GstPriQueue * pq) +{ + GstPriQueueNode *node, *min_node; + + node = pq->head; + if (!node) + return NULL; + + min_node = node; + for (node = node->next; node; node = node->next) { + if (compare_nodes (pq, node, min_node) < 0) + min_node = node; + } + + return min_node; +} + +/* + * Debug helper functions + */ + +static void +write_dot_node (const GstPriQueueNode * node, FILE * out, + GstPriQueueWriteElem write_elem_func, gpointer user_data) +{ + fprintf (out, " %" G_GUINTPTR_FORMAT " [label=\"", (guintptr) node); + write_elem_func (out, NODE2ELEM (node), user_data); + fprintf (out, "\"];\n"); +} + +static void +_write_dot_children (const GstPriQueue * pq, GstPriQueueNode * root, FILE * out, + GstPriQueueWriteElem write_elem_func, gpointer user_data) +{ + GstPriQueueNode *node; + + for (node = root->children_head; node; node = node->next) { + write_dot_node (node, out, write_elem_func, user_data); + + fprintf (out, " %" G_GUINTPTR_FORMAT, (guintptr) root); + fprintf (out, " -> %" G_GUINTPTR_FORMAT, (guintptr) node); + fprintf (out, " [color=red];\n"); + + if (node->parent) { + fprintf (out, " %" G_GUINTPTR_FORMAT, (guintptr) node); + fprintf (out, " -> %" G_GUINTPTR_FORMAT, (guintptr) node->parent); + fprintf (out, " [color=blue];\n"); + } + + if (node->children_head) + _write_dot_children (pq, node, out, write_elem_func, user_data); + } +} + +static void +_write_dot_tree (const GstPriQueue * pq, GstPriQueueNode * tree, FILE * out, + GstPriQueueWriteElem write_elem_func, gpointer user_data) +{ + /* Binomial heap list (list of root binomial trees) is in increasing order, + * but we want the trees to be in decreasing order (from left to right) in the + * DOT file, so recurse to get a kind of post-order traversal. + */ + if (tree->next) + _write_dot_tree (pq, tree->next, out, write_elem_func, user_data); + + write_dot_node (tree, out, write_elem_func, user_data); + _write_dot_children (pq, tree, out, write_elem_func, user_data); +} + +static gboolean +_binom_tree_is_invariant (const GstPriQueue * pq, GstPriQueueNode * root, + gsize * size) +{ + GstPriQueueNode *child; + gsize subtree_size; + gint res, num_children, expected_order; + + *size = 1; + num_children = 0; + expected_order = root->order; + + for (child = root->children_head; child; child = child->next) { + num_children++; + expected_order--; + + if (G_UNLIKELY (child->order != expected_order)) + return FALSE; + + res = compare_nodes (pq, root, child); + if (G_UNLIKELY (res > 0)) + return FALSE; + + if (G_UNLIKELY (child->parent != root)) + return FALSE; + + if (G_UNLIKELY (!_binom_tree_is_invariant (pq, child, &subtree_size))) + return FALSE; + + *size += subtree_size; + } + + if (G_UNLIKELY (expected_order != 0)) + return FALSE; + + if (G_UNLIKELY (num_children != root->order)) + return FALSE; + + return TRUE; +} + +static inline gboolean +is_heap_list_order_increasing (const GstPriQueueNode * head) +{ + const GstPriQueueNode *node; + + for (node = head; node && node->next; node = node->next) { + if (node->order >= node->next->order) + return FALSE; + } + + return TRUE; +} + +/* + * Public API + */ + +/** + * gst_pri_queue_create: (skip) + * @cmp_func: the #GstPriQueueCompareFunc used to compare elements in the + * #GstPriQueue. It should return a number < 0 if the first element is smaller + * (i.e. has a higher priority) than the second. + * @user_data: user data passed to @cmp_func + * + * Creates a new #GstPriQueue. The @cmp_func is used to determine which of any + * two elements in the #GstPriQueue is smaller (i.e. has the highest priority). + * + * Returns: a new #GstPriQueue + */ + /** + * GstPriQueueCompareFunc: (skip) + * @elem_a: a #GstPriQueueElem + * @elem_b: a #GstPriQueueElem to compare with + * @user_data: user data + * + * Specifies the type of a comparison function used to compare two elements in + * the #GstPriQueue. The function should return a negative integer if the first + * element is smaller (i.e. has a higher priority) than the second, 0 if they + * are equal, or a positive integer if the first element is larger (i.e. has a + * lower priority). + * + * Returns: negative value if @elem_a < @elem_b ; zero if @elem_a = @elem_b ; + * positive value if @elem_a > @elem_b + */ +GstPriQueue * +gst_pri_queue_create (GstPriQueueCompareFunc cmp_func, gpointer user_data) +{ + GstPriQueue *pq; + + pq = g_new (GstPriQueue, 1); + pq->cmp_func = cmp_func; + pq->user_data = user_data; + pq->head = NULL; + pq->size = 0; + + return pq; +} + +/** + * gst_pri_queue_destroy: (skip) + * @pq: the #GstPriQueue to destroy + * @elem_destroy_func: (nullable): a function that is called for each + * element in the #GstPriQueue before it is destroyed (e.g. to free the memory + * allocated for the element). + * + * Destroys the #GstPriQueue. + */ +void +gst_pri_queue_destroy (GstPriQueue * pq, GDestroyNotify elem_destroy_func) +{ + GstPriQueueElem *elem; + GstPriQueueIter iter; + + gst_pri_queue_iter_init (&iter, pq); + while (gst_pri_queue_iter_next (&iter, &elem)) { + if (elem_destroy_func) + elem_destroy_func (elem); + else + init_node (ELEM2NODE (elem)); + } + + g_free (pq); +} + +/** + * gst_pri_queue_size: (skip) + * @pq: a #GstPriQueue + * + * Returns the number of elements in the #GstPriQueue. This function is + * implemented in `O(1)` time. + * + * Returns: the number of elements in the #GstPriQueue + */ +gsize +gst_pri_queue_size (const GstPriQueue * pq) +{ + return pq->size; +} + +/** + * gst_pri_queue_insert: (skip) + * @pq: a #GstPriQueue + * @elem: the #GstPriQueueElem to be inserted + * + * Inserts an element into the #GstPriQueue. This function should only be called + * for elements that are not already inserted into the #GstPriQueue. + */ +void +gst_pri_queue_insert (GstPriQueue * pq, GstPriQueueElem * elem) +{ + GstPriQueueNode *node; + + node = ELEM2NODE (elem); + init_node (node); + binom_heap_list_add_node (pq, &pq->head, node); + pq->size++; +} + +/** + * gst_pri_queue_remove: (skip) + * @pq: a #GstPriQueue + * @elem: the #GstPriQueueElem to remove + * + * Removes a #GstPriQueueElem from the #GstPriQueue. This function should only + * be called for elements that are inserted into the #GstPriQueue. + */ +void +gst_pri_queue_remove (GstPriQueue * pq, GstPriQueueElem * elem) +{ + GstPriQueueNode *delnode; + + delnode = ELEM2NODE (elem); + (void) decrease_key (pq, delnode, TRUE); + remove_heap_root (pq, delnode); + pq->size--; + + init_node (delnode); +} + +/** + * gst_pri_queue_update: (skip) + * @pq: a #GstPriQueue + * @elem: a #GstPriQueueElem that has changed its associated value (priority) + * + * Updates an element's position in the #GstPriQueue after its value (priority) + * has changed such that the priority queue's comparison function may return a + * different value when comparing it to other elements. This function should + * only be called for elements that are inserted into the #GstPriQueue. + * + * Changing the value associated with a #GstPriQueueElem without calling + * gst_pri_queue_update() afterwards will most likely lead to the #GstPriQueue + * becoming invalid. + */ +void +gst_pri_queue_update (GstPriQueue * pq, GstPriQueueElem * elem) +{ + GstPriQueueNode *node; + + node = ELEM2NODE (elem); + if (!decrease_key (pq, node, FALSE)) + increase_key (pq, node); +} + +/** + * gst_pri_queue_get_min: (skip) + * @pq: a #GstPriQueue + * + * Returns the #GstPriQueueElem with the smallest value (i.e highest priority) + * without dequeuing it from the #GstPriQueue. If the #GstPriQueue contains + * multiple elements with the same (smallest) value, the returned + * #GstPriQueueElem will be one of these. Two consecutive calls to + * gst_pri_queue_get_min() are guaranteed to return the same #GstPriQueueElem + * if the #GstPriQueue has not been modified in between the calls. + * + * Returns: the smallest #GstPriQueueElem in the #GstPriQueue, or %NULL if the + * #GstPriQueue is empty. + */ +GstPriQueueElem * +gst_pri_queue_get_min (const GstPriQueue * pq) +{ + GstPriQueueNode *node; + + node = get_min_root (pq); + return NODE2ELEM (node); +} + +/** + * gst_pri_queue_pop_min: (skip) + * @pq: a #GstPriQueue + * + * Dequeues the #GstPriQueueElem with the smallest value (i.e highest priority) + * from the #GstPriQueue. If the #GstPriQueue contains multiple elements with + * the same (smallest) value, the dequeued #GstPriQueueElem will be one of + * these. + * + * Returns: the smallest #GstPriQueueElem in the #GstPriQueue, or %NULL if the + * #GstPriQueue is empty. + */ +GstPriQueueElem * +gst_pri_queue_pop_min (GstPriQueue * pq) +{ + GstPriQueueNode *delnode; + + delnode = get_min_root (pq); + if (!delnode) + return NULL; + + remove_heap_root (pq, delnode); + pq->size--; + + init_node (delnode); + return NODE2ELEM (delnode); +} + +/** + * gst_pri_queue_meld: (skip) + * @pqa: a #GstPriQueue + * @pqb: a #GstPriQueue to merge with + * + * Merge two priority queues. The second #GstPriQueue @pqb will be destroyed, + * and all its elements will be inserted into the first #GstPriQueue @pqa. This + * function is implemented in `O(log n)` time. + * + * Returns: @pqa after the elements from @pqb have been merged into it. + */ +GstPriQueue * +gst_pri_queue_meld (GstPriQueue * pqa, GstPriQueue * pqb) +{ + binom_heap_union (pqa, pqb->head); + pqa->size += pqb->size; + + pqb->head = NULL; + gst_pri_queue_destroy (pqb, NULL); + + return pqa; +} + +/* + * Iterator API + */ + +/** + * gst_pri_queue_iter_init: (skip) + * @iter: a #GstPriQueueIter + * @pq: a #GstPriQueue + * + * Initialize an iterator over the elements in @pq. + * + * Note that the iterator becomes invalid if the #GstPriQueue @pq is modified. + */ +void +gst_pri_queue_iter_init (GstPriQueueIter * iter, GstPriQueue * pq) +{ + iter->node = pq->head; +} + +/** + * gst_pri_queue_iter_next: (skip) + * @iter: a #GstPriQueueIter + * @elem: (out) (nullable) (optional): a location to store the #GstPriQueueElem, + * or %NULL + * + * Retrieves the #GstPriQueueElem at the iterator's current position, and + * advances the iterator. + * + * Example: + * |[ + * GstPriQueueIter iter; + * GstPriQueueElem *elem; + * + * gst_pri_queue_iter_init (&iter, pq); + * while (gst_pri_queue_iter_next (&iter, &elem)) { + * SomeValue *value = GST_CONTAINER_OF (elem, SomeValue, pq_elem); + * ... + * } + * ]| + * + * Note that iteration over elements in a #GstPriQueue is performed in an + * arbitrary order, and not in priority order. The first + * element of an iteration sequence is not even guaranteed to be the smallest + * element returned by gst_pri_queue_get_min() or gst_pri_queue_pop_min(), or + * any of the smallest elements in the #GstPriQueue if there are multiple such. + * + * Returns: %TRUE if a #GstPriQueueElem could be retrieved, or %FALSE if the end + * of the #GstPriQueue has been reached. + */ +gboolean +gst_pri_queue_iter_next (GstPriQueueIter * iter, GstPriQueueElem ** elem) +{ + GstPriQueueNode *node, *next; + + node = iter->node; + if (!node) + return FALSE; + + if (node->children_head) { + next = node->children_head; + } else { + for (next = node; next && !next->next;) + next = next->parent; + + if (next) + next = next->next; + } + + iter->node = next; + if (elem) + *elem = NODE2ELEM (node); + + return TRUE; +} + +/* + * Debug API + */ + +/** + * gst_pri_queue_is_valid: (skip) + * @pq: a #GstPriQueue + * + * Check that the invariants of the internal data structures have not been + * violated. + * + * This function is intended for testing and/or debugging purposes only. + * + * + * Returns: %TRUE if no invariants have been violated, else %FALSE. + */ +gboolean +gst_pri_queue_is_valid (const GstPriQueue * pq) +{ + GstPriQueueNode *heap; + gsize size, heap_size; + + if (G_UNLIKELY (!is_heap_list_order_increasing (pq->head))) + return FALSE; + + size = 0; + for (heap = pq->head; heap; heap = heap->next) { + if (G_UNLIKELY (!_binom_tree_is_invariant (pq, heap, &heap_size))) + return FALSE; + + size += heap_size; + + if (G_UNLIKELY (heap->parent != NULL)) + return FALSE; + } + + if (G_UNLIKELY (size != pq->size)) + return FALSE; + + return TRUE; +} + +/** + * gst_pri_queue_write_dot_file: (skip) + * @pq: a #GstPriQueue + * @out: the output stream to which the DOT file is written + * @write_elem_func: a #GstPriQueueWriteElem used as callback to write elements + * to the DOT file + * @user_data: user data passed to @write_elem_func + * + * Write a DOT representation of the internal data structures to the given + * output stream. + * + * This function is intended for testing and/or debugging purposes only. + * + */ + /** + * GstPriQueueWriteElem: (skip) + * @out: the #FILE to where the element is written + * @elem: the element to write + * @user_data: user data + * + * A #GstPriQueueWriteElem is a callback supplied to + * gst_pri_queue_write_dot_file() for writing elements to a DOT file. + * + * Returns: the number of bytes written. + */ +void +gst_pri_queue_write_dot_file (const GstPriQueue * pq, FILE * out, + GstPriQueueWriteElem write_elem_func, gpointer user_data) +{ + fprintf (out, "digraph graphname {\n"); + + if (pq->head) + _write_dot_tree (pq, pq->head, out, write_elem_func, user_data); + + fprintf (out, "}\n"); +} diff --git a/subprojects/gstreamer/libs/gst/base/gstpriqueue.h b/subprojects/gstreamer/libs/gst/base/gstpriqueue.h new file mode 100644 index 00000000000..d5e54766dde --- /dev/null +++ b/subprojects/gstreamer/libs/gst/base/gstpriqueue.h @@ -0,0 +1,124 @@ +/* GStreamer + * Copyright (C) 2016 Pexip AS + * Erlend Graff + * + * gstpriqueue.h: binomial heap implementation of a priority queue + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_PRIQUEUE_H__ +#define __GST_PRIQUEUE_H__ + +#include +#include + +G_BEGIN_DECLS + +typedef struct _GstPriQueue GstPriQueue; +typedef struct _GstPriQueueElem GstPriQueueElem; +typedef struct _GstPriQueueIter GstPriQueueIter; + +typedef gint (* GstPriQueueCompareFunc) (const GstPriQueueElem * elem_a, + const GstPriQueueElem * elem_b, + gpointer user_data); + +struct _GstPriQueueNode +{ + /*< private >*/ + struct _GstPriQueueNode *parent; + struct _GstPriQueueNode *children_head; + struct _GstPriQueueNode *next; + gint order; +}; + +struct _GstPriQueueElem +{ + /*< private >*/ + struct _GstPriQueueNode node; +}; + +struct _GstPriQueueIter +{ + /*< private >*/ + struct _GstPriQueueNode *node; +}; + +GstPriQueue * +gst_pri_queue_create (GstPriQueueCompareFunc cmp_func, + gpointer user_data); + +void +gst_pri_queue_destroy (GstPriQueue * pq, + GDestroyNotify elem_destroy_func); + +gsize +gst_pri_queue_size (const GstPriQueue * pq); + +void +gst_pri_queue_insert (GstPriQueue * pq, + GstPriQueueElem * elem); + +void +gst_pri_queue_remove (GstPriQueue * pq, + GstPriQueueElem * elem); + +void +gst_pri_queue_update (GstPriQueue * pq, + GstPriQueueElem * elem); + +GstPriQueueElem * +gst_pri_queue_get_min (const GstPriQueue * pq); + +GstPriQueueElem * +gst_pri_queue_pop_min (GstPriQueue * pq); + +GstPriQueue * +gst_pri_queue_meld (GstPriQueue * pqa, + GstPriQueue * pqb); + +/* + * Iterator API + */ + +void +gst_pri_queue_iter_init (GstPriQueueIter * iter, + GstPriQueue * pq); + +gboolean +gst_pri_queue_iter_next (GstPriQueueIter * iter, + GstPriQueueElem ** elem); + +/* + * Debug API + */ + +typedef gint (* GstPriQueueWriteElem) (FILE * out, + const GstPriQueueElem * elem, + gpointer user_data); + +gboolean +gst_pri_queue_is_valid (const GstPriQueue * pq); + +void +gst_pri_queue_write_dot_file (const GstPriQueue * pq, + FILE * out, + GstPriQueueWriteElem write_elem_func, + gpointer user_data); + +G_END_DECLS + +#endif /* __GST_PRIQUEUE_H__ */ diff --git a/subprojects/gstreamer/libs/gst/base/meson.build b/subprojects/gstreamer/libs/gst/base/meson.build index e08254d929c..64292f15b89 100644 --- a/subprojects/gstreamer/libs/gst/base/meson.build +++ b/subprojects/gstreamer/libs/gst/base/meson.build @@ -4,6 +4,7 @@ gst_base_sources = files( 'gstbaseparse.c', 'gstbasesink.c', 'gstbasesrc.c', + 'gstbaseidlesrc.c', 'gstbasetransform.c', 'gstbitreader.c', 'gstbitwriter.c', @@ -12,6 +13,7 @@ gst_base_sources = files( 'gstcollectpads.c', 'gstdataqueue.c', 'gstflowcombiner.c', + 'gstpriqueue.c', 'gstpushsrc.c', 'gstqueuearray.c', 'gsttypefindhelper.c', @@ -25,6 +27,7 @@ gst_base_headers = files( 'gstbaseparse.h', 'gstbasesink.h', 'gstbasesrc.h', + 'gstbaseidlesrc.h', 'gstbasetransform.h', 'gstbitreader.h', 'gstbitwriter.h', @@ -33,6 +36,7 @@ gst_base_headers = files( 'gstcollectpads.h', 'gstdataqueue.h', 'gstflowcombiner.h', + 'gstpriqueue.h', 'gstpushsrc.h', 'gstqueuearray.h', 'gsttypefindhelper.h', @@ -99,6 +103,7 @@ install_headers('base.h', 'gstbaseparse.h', 'gstbasesink.h', 'gstbasesrc.h', + 'gstbaseidlesrc.h', 'gstbasetransform.h', 'gstbitreader.h', 'gstbitwriter.h', @@ -107,6 +112,7 @@ install_headers('base.h', 'gstcollectpads.h', 'gstdataqueue.h', 'gstflowcombiner.h', + 'gstpriqueue.h', 'gstpushsrc.h', 'gstqueuearray.h', 'gsttypefindhelper.h', diff --git a/subprojects/gstreamer/libs/gst/check/gstcheck.c b/subprojects/gstreamer/libs/gst/check/gstcheck.c index 3ff00be8b15..9e77afb9d06 100644 --- a/subprojects/gstreamer/libs/gst/check/gstcheck.c +++ b/subprojects/gstreamer/libs/gst/check/gstcheck.c @@ -328,7 +328,9 @@ print_plugins (void) static void gst_check_deinit (void) { +#if 0 gst_deinit (); +#endif gst_check_clear_log_filter (); } diff --git a/subprojects/gstreamer/libs/gst/check/gstharness.c b/subprojects/gstreamer/libs/gst/check/gstharness.c index 03a135f49e8..9076eca02a5 100644 --- a/subprojects/gstreamer/libs/gst/check/gstharness.c +++ b/subprojects/gstreamer/libs/gst/check/gstharness.c @@ -1,6 +1,6 @@ /* GstHarness - A test-harness for GStreamer testing * - * Copyright (C) 2012-2015 Pexip + * Copyright (C) 2012-2021 Pexip * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -132,7 +132,12 @@ #include #include +GST_DEBUG_CATEGORY (gst_harness_debug); +#define GST_CAT_DEFAULT gst_harness_debug + static void gst_harness_stress_free (GstHarnessThread * t); +static GstBuffer *gst_harness_extract_one_buffer_locked (GstHarness * h, + GstMiniObject * queue_element); /* Keys used for storing and retrieving associations to pads and elements with * g_object_set_data (): */ @@ -211,6 +216,8 @@ struct _GstHarnessPrivate gboolean eos_received; GPtrArray *stress; + + guint64 pull_timeout; }; static GstFlowReturn @@ -229,6 +236,10 @@ gst_harness_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) g_mutex_lock (&priv->blocking_push_mutex); g_atomic_int_inc (&priv->recv_buffers); + GST_TRACE_OBJECT (pad, + "receiving chain buffer: harness: %" GST_PTR_FORMAT " buf: %" + GST_PTR_FORMAT, h, buffer); + if (priv->drop_buffers) { gst_buffer_unref (buffer); } else { @@ -248,6 +259,47 @@ gst_harness_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer) return GST_FLOW_OK; } +static GstFlowReturn +gst_harness_chain_list (GstPad * pad, GstObject * parent, + GstBufferList * buffer_list) +{ + GstHarness *h; + GstHarnessLink *link; + GstHarnessPrivate *priv; + guint listlen; + (void) parent; + + if (!(link = gst_harness_pad_link_lock (pad, &h))) + return GST_FLOW_FLUSHING; + g_assert (h != NULL); + priv = h->priv; + + g_mutex_lock (&priv->blocking_push_mutex); + listlen = gst_buffer_list_length (buffer_list); + g_atomic_int_add (&priv->recv_buffers, listlen); + + GST_TRACE_OBJECT (pad, + "receiving chain list harness: %" GST_PTR_FORMAT " #buffers: (%" + G_GUINT32_FORMAT ") %" GST_PTR_FORMAT, h, listlen, buffer_list); + + if (priv->drop_buffers) { + gst_buffer_list_unref (buffer_list); + } else { + g_mutex_lock (&priv->buf_or_eos_mutex); + g_async_queue_push (priv->buffer_queue, buffer_list); + g_cond_signal (&priv->buf_or_eos_cond); + g_mutex_unlock (&priv->buf_or_eos_mutex); + } + + if (priv->blocking_push_mode) { + g_cond_wait (&priv->blocking_push_cond, &priv->blocking_push_mutex); + } + g_mutex_unlock (&priv->blocking_push_mutex); + + gst_harness_link_unlock (link); + return GST_FLOW_OK; +} + static gboolean gst_harness_src_event (GstPad * pad, GstObject * parent, GstEvent * event) { @@ -643,6 +695,7 @@ gst_harness_setup_sink_pad (GstHarness * h, gst_harness_pad_link_set (h->sinkpad, h); gst_pad_set_chain_function (h->sinkpad, gst_harness_chain); + gst_pad_set_chain_list_function (h->sinkpad, gst_harness_chain_list); gst_pad_set_query_function (h->sinkpad, gst_harness_sink_query); gst_pad_set_event_function (h->sinkpad, gst_harness_sink_event); @@ -719,6 +772,8 @@ gst_harness_new_empty (void) GstHarness *h; GstHarnessPrivate *priv; + GST_DEBUG_CATEGORY_INIT (gst_harness_debug, "gstharness", 0, "GstHarness"); + h = g_new0 (GstHarness, 1); g_assert (h != NULL); h->priv = g_new0 (GstHarnessPrivate, 1); @@ -733,7 +788,7 @@ gst_harness_new_empty (void) priv->testclock = GST_TEST_CLOCK_CAST (gst_test_clock_new ()); priv->buffer_queue = g_async_queue_new_full ( - (GDestroyNotify) gst_buffer_unref); + (GDestroyNotify) gst_mini_object_unref); priv->src_event_queue = g_async_queue_new_full ( (GDestroyNotify) gst_event_unref); priv->sink_event_queue = g_async_queue_new_full ( @@ -756,6 +811,9 @@ gst_harness_new_empty (void) /* we have forwarding on as a default */ gst_harness_set_forwarding (h, TRUE); + /* we default to 60s as pull timeout */ + gst_harness_set_pull_timeout (h, G_USEC_PER_SEC * 60); + return h; } @@ -1087,6 +1145,7 @@ gst_harness_new_parse (const gchar * launchline) return h; } + /** * gst_harness_teardown: * @h: a #GstHarness @@ -1122,8 +1181,11 @@ gst_harness_teardown (GstHarness * h) h->sink_harness = NULL; if (h->srcpad) { - if (gst_pad_is_request_pad (GST_PAD_PEER (h->srcpad))) - gst_element_release_request_pad (h->element, GST_PAD_PEER (h->srcpad)); + GstPad *peer_srcpad = gst_pad_get_peer (h->srcpad); + if (gst_pad_is_request_pad (peer_srcpad)) + gst_element_release_request_pad (h->element, peer_srcpad); + if (peer_srcpad) + gst_object_unref (peer_srcpad); g_free (priv->element_sinkpad_name); gst_pad_set_active (h->srcpad, FALSE); @@ -1139,8 +1201,11 @@ gst_harness_teardown (GstHarness * h) h->srcpad = NULL; if (h->sinkpad) { - if (gst_pad_is_request_pad (GST_PAD_PEER (h->sinkpad))) - gst_element_release_request_pad (h->element, GST_PAD_PEER (h->sinkpad)); + GstPad *peer_sinkpad = gst_pad_get_peer (h->sinkpad); + if (gst_pad_is_request_pad (peer_sinkpad)) + gst_element_release_request_pad (h->element, peer_sinkpad); + if (peer_sinkpad) + gst_object_unref (peer_sinkpad); g_free (priv->element_srcpad_name); gst_pad_set_active (h->sinkpad, FALSE); @@ -1149,6 +1214,7 @@ gst_harness_teardown (GstHarness * h) * they try to access this harness through pad data */ gst_harness_pad_link_tear_down (h->sinkpad); gst_pad_set_chain_function (h->sinkpad, NULL); + gst_pad_set_chain_list_function (h->sinkpad, NULL); gst_pad_set_event_function (h->sinkpad, NULL); gst_pad_set_query_function (h->sinkpad, NULL); @@ -1636,6 +1702,31 @@ gst_harness_set_forward_pad (GstHarness * h, GstPad * fwdpad) (GstObject *) fwdpad); } +/** + * gst_harness_set_pull_timeout: + * @h: a #GstHarness + * @timeout: timeout in microseconds + * + * Sets the timeout to use on all pull-like methods. This affects src and sink + * harnesses if any. + * + * MT safe. + * + * Since: 1.20 + **/ +void +gst_harness_set_pull_timeout (GstHarness * h, guint64 timeout) +{ + GstHarnessPrivate *priv = h->priv; + priv->pull_timeout = timeout; + + /* propagate timeout to src and sink harnesses, if any */ + if (h->src_harness) + gst_harness_set_pull_timeout (h->src_harness, timeout); + if (h->sink_harness) + gst_harness_set_pull_timeout (h->sink_harness, timeout); +} + /** * gst_harness_create_buffer: * @h: a #GstHarness @@ -1704,13 +1795,40 @@ gst_harness_push (GstHarness * h, GstBuffer * buffer) return gst_pad_push (h->srcpad, buffer); } +/** + * gst_harness_push_list: + * @h: a #GstHarness + * @buffer_list: (transfer full): a #GstBufferList to push + * + * Pushes a #GstBufferList on the #GstHarness srcpad. The standard way of + * interacting with an harnessed element. + * + * MT safe. + * + * Returns: a #GstFlowReturn with the result from the push + * + */ +GstFlowReturn +gst_harness_push_list (GstHarness * h, GstBufferList * buffer_list) +{ + guint32 listlen; + GstHarnessPrivate *priv = h->priv; + g_assert (buffer_list != NULL); + listlen = gst_buffer_list_length (buffer_list); + if (listlen > 0) { + GstBuffer *last_buffer = gst_buffer_list_get (buffer_list, listlen - 1); + priv->last_push_ts = GST_BUFFER_TIMESTAMP (last_buffer); + } + return gst_pad_push_list (h->srcpad, buffer_list); +} + /** * gst_harness_pull: * @h: a #GstHarness * * Pulls a #GstBuffer from the #GAsyncQueue on the #GstHarness sinkpad. The pull - * will timeout in 60 seconds. This is the standard way of getting a buffer - * from a harnessed #GstElement. + * will timeout in #GstHarness.pull_timeout seconds. This is the standard way + * of getting a buffer from a harnessed #GstElement. * * MT safe. * @@ -1721,9 +1839,17 @@ gst_harness_push (GstHarness * h, GstBuffer * buffer) GstBuffer * gst_harness_pull (GstHarness * h) { - GstHarnessPrivate *priv = h->priv; - GstBuffer *buf = (GstBuffer *) g_async_queue_timeout_pop (priv->buffer_queue, - G_USEC_PER_SEC * 60); + GstHarnessPrivate *priv; + GstMiniObject *queue_element; + GstBuffer *buf; + + priv = h->priv; + queue_element = + GST_MINI_OBJECT_CAST (g_async_queue_timeout_pop (priv->buffer_queue, + priv->pull_timeout)); + g_mutex_lock (&priv->buf_or_eos_mutex); + buf = gst_harness_extract_one_buffer_locked (h, queue_element); + g_mutex_unlock (&priv->buf_or_eos_mutex); if (priv->blocking_push_mode) { g_mutex_lock (&priv->blocking_push_mutex); @@ -1751,13 +1877,15 @@ gst_harness_pull (GstHarness * h) gboolean gst_harness_pull_until_eos (GstHarness * h, GstBuffer ** buf) { + GstMiniObject *queue_element; GstHarnessPrivate *priv = h->priv; gboolean success = TRUE; gint64 end_time = g_get_monotonic_time () + 60 * G_TIME_SPAN_SECOND; g_mutex_lock (&priv->buf_or_eos_mutex); while (success) { - *buf = g_async_queue_try_pop (priv->buffer_queue); + queue_element = g_async_queue_try_pop (priv->buffer_queue); + *buf = gst_harness_extract_one_buffer_locked (h, queue_element); if (*buf || priv->eos_received) break; success = g_cond_wait_until (&priv->buf_or_eos_cond, @@ -1768,6 +1896,56 @@ gst_harness_pull_until_eos (GstHarness * h, GstBuffer ** buf) return success; } +static void +gst_harness_async_queue_transfer (GAsyncQueue * new, GAsyncQueue * old) +{ + gpointer popped; + while ((popped = g_async_queue_try_pop (old)) != NULL) { + g_async_queue_push (new, popped); + } +} + +static GstBuffer * +gst_harness_extract_one_buffer_locked (GstHarness * h, + GstMiniObject * queue_element) +{ + GstHarnessPrivate *priv; + GAsyncQueue *old_buffer_queue; + GstBuffer *buf = NULL; + priv = h->priv; + if (queue_element != NULL) { + if (GST_IS_BUFFER (queue_element)) { + buf = GST_BUFFER_CAST (queue_element); + } else { + guint32 listlen; + GstBufferList *list = GST_BUFFER_LIST_CAST (queue_element); + g_assert (GST_BUFFER_LIST (queue_element)); + + listlen = gst_buffer_list_length (list); + if (listlen > 0) { + buf = gst_buffer_ref (gst_buffer_list_get (list, 0)); + } + if (listlen > 1) { + /* List has multiple elements, remove the first one and transfer the + rest of the list as separate buffers to a new queue */ + gst_buffer_list_remove (list, 0, 1); + listlen--; + old_buffer_queue = priv->buffer_queue; + priv->buffer_queue = g_async_queue_new_full ( + (GDestroyNotify) gst_mini_object_unref); + for (int i = 0; i < listlen; ++i) { + g_async_queue_push (priv->buffer_queue, + gst_buffer_ref (gst_buffer_list_get (list, i))); + } + gst_harness_async_queue_transfer (priv->buffer_queue, old_buffer_queue); + g_async_queue_unref (old_buffer_queue); + } + gst_buffer_list_unref (list); + } + } + return buf; +} + /** * gst_harness_try_pull: * @h: a #GstHarness @@ -1785,8 +1963,20 @@ gst_harness_pull_until_eos (GstHarness * h, GstBuffer ** buf) GstBuffer * gst_harness_try_pull (GstHarness * h) { - GstHarnessPrivate *priv = h->priv; - GstBuffer *buf = (GstBuffer *) g_async_queue_try_pop (priv->buffer_queue); + GstHarnessPrivate *priv; + GstBuffer *buf; + GstMiniObject *queue_element; + + g_return_val_if_fail (h != NULL, NULL); + + priv = h->priv; + + queue_element = + GST_MINI_OBJECT_CAST (g_async_queue_try_pop (priv->buffer_queue)); + + g_mutex_lock (&priv->buf_or_eos_mutex); + buf = gst_harness_extract_one_buffer_locked (h, queue_element); + g_mutex_unlock (&priv->buf_or_eos_mutex); if (priv->blocking_push_mode) { g_mutex_lock (&priv->blocking_push_mutex); @@ -1855,8 +2045,27 @@ gst_harness_buffers_received (GstHarness * h) guint gst_harness_buffers_in_queue (GstHarness * h) { + gpointer popped; + GAsyncQueue *old_buffer_queue; GstHarnessPrivate *priv = h->priv; - return g_async_queue_length (priv->buffer_queue); + guint enqueued_buffers = 0; + + g_mutex_lock (&priv->buf_or_eos_mutex); + old_buffer_queue = priv->buffer_queue; + priv->buffer_queue = g_async_queue_new_full ( + (GDestroyNotify) gst_mini_object_unref); + while ((popped = g_async_queue_try_pop (old_buffer_queue)) != NULL) { + if (GST_IS_BUFFER_LIST (popped)) { + GstBufferList *popped_list = GST_BUFFER_LIST_CAST (popped); + enqueued_buffers += gst_buffer_list_length (popped_list); + } else { + enqueued_buffers++; + } + g_async_queue_push (priv->buffer_queue, popped); + } + g_async_queue_unref (old_buffer_queue); + g_mutex_unlock (&priv->buf_or_eos_mutex); + return enqueued_buffers; } /** @@ -1894,6 +2103,7 @@ gst_harness_take_all_data_as_buffer (GstHarness * h) { GstHarnessPrivate *priv; GstBuffer *ret, *buf; + GstMiniObject *queue_element; g_return_val_if_fail (h != NULL, NULL); @@ -1901,16 +2111,23 @@ gst_harness_take_all_data_as_buffer (GstHarness * h) g_async_queue_lock (priv->buffer_queue); - ret = g_async_queue_try_pop_unlocked (priv->buffer_queue); + queue_element = g_async_queue_try_pop_unlocked (priv->buffer_queue); - if (ret == NULL) { + if (queue_element == NULL) { ret = gst_buffer_new (); } else { + /* Extracting one element and pushing the rest to a queue inside the same + function is not very efficient, but we are not efficient in appending + buffers either, so let's stick with the KISS theme. */ + ret = gst_harness_extract_one_buffer_locked (h, queue_element); /* buffer appending isn't very efficient for larger numbers of buffers * or lots of memories, but this function is not performance critical and * we can still improve it if and when the need arises. For now KISS. */ - while ((buf = g_async_queue_try_pop_unlocked (priv->buffer_queue))) + while ((queue_element = + g_async_queue_try_pop_unlocked (priv->buffer_queue))) { + buf = gst_harness_extract_one_buffer_locked (h, queue_element); ret = gst_buffer_append (ret, buf); + } } g_async_queue_unlock (priv->buffer_queue); @@ -1971,6 +2188,113 @@ gst_harness_take_all_data_as_bytes (GstHarness * h) } +/** + * gst_harness_pull_list + * @h: a #GstHarness + * + * Pulls a #GstBufferList from the #GAsyncQueue on the #GstHarness sinkpad. + * The pull will timeout in 60 seconds. This is the standard way of getting + * a buffer list from a harnessed #GstElement. + * + * If a normal buffer is submitted to the GstHarness this function will return + * null as long as that buffer is at the head of the queue + * (use #gst_harness_pull to remove it) + * + * MT safe. + * + * Returns: (transfer full): a #GstBufferList or %NULL if timed out, + * or buffer lists are not enabled. + */ +GstBufferList * +gst_harness_pull_list (GstHarness * h) +{ + GAsyncQueue *old_buffer_queue; + GstHarnessPrivate *priv; + GstMiniObject *queue_element; + g_return_val_if_fail (h != NULL, NULL); + priv = h->priv; + + queue_element = + GST_MINI_OBJECT_CAST (g_async_queue_timeout_pop (priv->buffer_queue, + G_USEC_PER_SEC * 60)); + g_mutex_lock (&priv->buf_or_eos_mutex); + if (queue_element != NULL && GST_IS_BUFFER (queue_element)) { + /* Head of queue is a buffer, add it back to the queue */ + old_buffer_queue = priv->buffer_queue; + priv->buffer_queue = g_async_queue_new_full ( + (GDestroyNotify) gst_mini_object_unref); + g_async_queue_push (priv->buffer_queue, queue_element); + gst_harness_async_queue_transfer (priv->buffer_queue, old_buffer_queue); + queue_element = NULL; + } + g_mutex_unlock (&priv->buf_or_eos_mutex); + + g_assert (queue_element == NULL || GST_IS_BUFFER_LIST (queue_element)); + + if (priv->blocking_push_mode) { + g_mutex_lock (&priv->blocking_push_mutex); + g_cond_signal (&priv->blocking_push_cond); + g_mutex_unlock (&priv->blocking_push_mutex); + } + + return GST_BUFFER_LIST_CAST (queue_element); +} + +/** + * gst_harness_try_pull_list: + * @h: a #GstHarness + * + * Pulls a #GstBuffer from the #GAsyncQueue on the #GstHarness sinkpad. Unlike + * gst_harness_pull this will not wait for any buffers if not any are present, + * and return %NULL straight away. + * + * Pulls a #GstBufferList from the #GAsyncQueue on the #GstHarness sinkpad. + * Unlike gst_harness_pull_list this will not wait for any buffers if not any + * are present, and return %NULL straight away. + * This function is only relevant if #gst_harness_buffer_list_support_set_enable + * is set to true, otherwise NULL will be returned. + * + * MT safe. + * + * Returns: (transfer full): a #GstBuffer or %NULL if no buffers are present in + * the #GAsyncQueue, or buffer lists not enabled. + * + */ +GstBufferList * +gst_harness_try_pull_list (GstHarness * h) +{ + GAsyncQueue *old_buffer_queue; + GstMiniObject *queue_element; + GstHarnessPrivate *priv; + g_return_val_if_fail (h != NULL, NULL); + priv = h->priv; + + g_mutex_lock (&priv->buf_or_eos_mutex); + + queue_element = g_async_queue_try_pop (priv->buffer_queue); + + if (queue_element != NULL && GST_IS_BUFFER (queue_element)) { + /* Head of queue is a buffer, add it back to the queue */ + old_buffer_queue = priv->buffer_queue; + priv->buffer_queue = g_async_queue_new_full ( + (GDestroyNotify) gst_mini_object_unref); + g_async_queue_push (priv->buffer_queue, queue_element); + gst_harness_async_queue_transfer (priv->buffer_queue, old_buffer_queue); + g_async_queue_unref (old_buffer_queue); + queue_element = NULL; + } + + g_mutex_unlock (&priv->buf_or_eos_mutex); + + if (priv->blocking_push_mode) { + g_mutex_lock (&priv->blocking_push_mutex); + g_cond_signal (&priv->blocking_push_cond); + g_mutex_unlock (&priv->blocking_push_mutex); + } + + return GST_BUFFER_LIST_CAST (queue_element); +} + /** * gst_harness_dump_to_file: * @h: a #GstHarness @@ -2043,7 +2367,7 @@ gst_harness_push_event (GstHarness * h, GstEvent * event) * @h: a #GstHarness * * Pulls an #GstEvent from the #GAsyncQueue on the #GstHarness sinkpad. - * Timeouts after 60 seconds similar to gst_harness_pull. + * Timeouts after #GstHarness.pull_timeout seconds similar to gst_harness_pull. * * MT safe. * @@ -2056,7 +2380,7 @@ gst_harness_pull_event (GstHarness * h) { GstHarnessPrivate *priv = h->priv; return (GstEvent *) g_async_queue_timeout_pop (priv->sink_event_queue, - G_USEC_PER_SEC * 60); + priv->pull_timeout); } /** @@ -2146,7 +2470,7 @@ gst_harness_push_upstream_event (GstHarness * h, GstEvent * event) * @h: a #GstHarness * * Pulls an #GstEvent from the #GAsyncQueue on the #GstHarness srcpad. - * Timeouts after 60 seconds similar to gst_harness_pull. + * Timeouts after #GstHarness.pull_timeout seconds similar to gst_harness_pull. * * MT safe. * @@ -2159,7 +2483,7 @@ gst_harness_pull_upstream_event (GstHarness * h) { GstHarnessPrivate *priv = h->priv; return (GstEvent *) g_async_queue_timeout_pop (priv->src_event_queue, - G_USEC_PER_SEC * 60); + priv->pull_timeout); } /** diff --git a/subprojects/gstreamer/libs/gst/check/gstharness.h b/subprojects/gstreamer/libs/gst/check/gstharness.h index 160fdb01d8e..633bd6dc4bc 100644 --- a/subprojects/gstreamer/libs/gst/check/gstharness.h +++ b/subprojects/gstreamer/libs/gst/check/gstharness.h @@ -174,6 +174,9 @@ void gst_harness_set_blocking_push_mode (GstHarness * h); GST_CHECK_API void gst_harness_set_forwarding (GstHarness * h, gboolean forwarding); +GST_CHECK_API +void gst_harness_set_pull_timeout (GstHarness * h, guint64 timeout); + /* buffers */ GST_CHECK_API @@ -182,6 +185,9 @@ GstBuffer * gst_harness_create_buffer (GstHarness * h, gsize size); GST_CHECK_API GstFlowReturn gst_harness_push (GstHarness * h, GstBuffer * buffer); +GST_CHECK_API +GstFlowReturn gst_harness_push_list (GstHarness * h, GstBufferList * buffer_list); + GST_CHECK_API GstBuffer * gst_harness_pull (GstHarness * h); @@ -218,6 +224,12 @@ GBytes * gst_harness_take_all_data_as_bytes (GstHarness * h); GST_CHECK_API GstClockTime gst_harness_get_last_pushed_timestamp (GstHarness * h); +GST_CHECK_API +GstBufferList * gst_harness_pull_list (GstHarness * h); + +GST_CHECK_API +GstBufferList *gst_harness_try_pull_list (GstHarness * h); + /* downstream events */ GST_CHECK_API diff --git a/subprojects/gstreamer/libs/gst/check/libcheck/check_run.c b/subprojects/gstreamer/libs/gst/check/libcheck/check_run.c index 2b8a87f5423..0ce9c6473f3 100644 --- a/subprojects/gstreamer/libs/gst/check/libcheck/check_run.c +++ b/subprojects/gstreamer/libs/gst/check/libcheck/check_run.c @@ -106,7 +106,7 @@ sig_handler (int sig_nr) switch (sig_nr) { case SIGALRM: alarm_received = 1; - killpg (group_pid, SIGKILL); + killpg (group_pid, SIGABRT); break; case SIGTERM: case SIGINT: @@ -157,6 +157,14 @@ srunner_run_end (SRunner * sr, enum print_output CK_ATTRIBUTE_UNUSED print_mode) set_fork_status (CK_FORK); } +static int +check_nofork_failure (SRunner * sr) +{ + return (srunner_fork_status (sr) == CK_NOFORK && + (sr->stats->n_errors + sr->stats->n_failed) > 0); +} + + static void srunner_iterate_suites (SRunner * sr, const char *sname, const char *tcname, @@ -205,6 +213,8 @@ srunner_iterate_suites (SRunner * sr, } srunner_run_tcase (sr, tc); + if (check_nofork_failure (sr)) + break; } log_suite_end (sr, s); @@ -252,8 +262,12 @@ srunner_iterate_tcase_tfuns (SRunner * sr, TCase * tc) if (NULL != tr) { srunner_add_failure (sr, tr); log_test_end (sr, tr); + if (check_nofork_failure (sr)) + break; } } + if (check_nofork_failure (sr)) + break; } } diff --git a/subprojects/gstreamer/libs/gst/check/libcheck/meson.build b/subprojects/gstreamer/libs/gst/check/libcheck/meson.build index af6210b0a05..2eb8659d9c9 100644 --- a/subprojects/gstreamer/libs/gst/check/libcheck/meson.build +++ b/subprojects/gstreamer/libs/gst/check/libcheck/meson.build @@ -40,7 +40,7 @@ if not cdata.has('HAVE_GETLINE') endif # FIXME: check that timer_create, timer_settime, timer_delete are in rt_lib -if not rt_lib.found() +if not rt_lib.found() or get_option('default_library') == 'static' libcheck_files += files( 'libcompat/timer_create.c', 'libcompat/timer_settime.c', diff --git a/subprojects/gstreamer/libs/gst/helpers/gst_gdb.py b/subprojects/gstreamer/libs/gst/helpers/gst_gdb.py index 90374fba976..1bf4674524e 100644 --- a/subprojects/gstreamer/libs/gst/helpers/gst_gdb.py +++ b/subprojects/gstreamer/libs/gst/helpers/gst_gdb.py @@ -423,7 +423,9 @@ def __str__(self): value = self.value() tname = self.fundamental_typename() gvalue_type = gdb.lookup_type("GValue") - if tname == "GstFraction": + if tname is None: + v = "" + elif tname == "GstFraction": v = "%d/%d" % (value[0], value[1]) elif tname == "GstBitmask": v = "0x%016x" % long(value) @@ -450,7 +452,7 @@ def __str__(self): v += " " if v == "<" else ", " v += str(GdbGValue(l)) v += " >" - elif tname in ("GEnum"): + elif tname == "GEnum": v = "%s(%s)" % ( g_type_to_name(g_type_to_typenode(self.val["g_type"])), value["v_int"]) @@ -527,7 +529,7 @@ def print(self, indent, prefix=None): else: _gdb_write(indent, "%s:" % (self.name())) for (key, value) in self.values(): - _gdb_write(indent+1, "%s: %s" % (key, str(value))) + _gdb_write(indent+1, "%s: %s (type %s)" % (key, str(value), value.fundamental_typename())) class GdbGstSegment: @@ -1094,24 +1096,31 @@ class GstDot(gdb.Command): """\ Create a pipeline dot file as close as possible to the output of GST_DEBUG_BIN_TO_DOT_FILE. This command will find the top-level parent -for the given gstreamer object and create the dot for that element. +for the given gstreamer object and create the dot representation +for that element or allow the creation of a 'local' dot representation +iff the given object is a valid bin and the optional argument 'local' +is provided in the argument list. -Usage: gst-dot """ +Usage: gst-dot (local)""" def __init__(self): super(GstDot, self).__init__("gst-dot", gdb.COMMAND_DATA) def invoke(self, arg, from_tty): self.dont_repeat() args = gdb.string_to_argv(arg) - if len(args) != 2: - raise Exception("Usage: gst-dot ") + if ((3 > len(args) < 2) + or (len(args) == 3 and args[2] != "local")): + raise Exception("Usage: gst-dot (local)") value = gdb.parse_and_eval(args[0]) if not value: raise Exception("'%s' is not a valid object" % args[0]) value = gst_object_from_value(value) - value = gst_object_pipeline(value) + if len(args) == 2: + value = gst_object_pipeline(value) + elif not gst_is_bin(value): + raise Exception("'%s' is not a valid bin" % value) dot = GdbGstElement(value).pipeline_dot() file = open(args[1], "w") diff --git a/subprojects/gstreamer/libs/gst/net/gstnet.h b/subprojects/gstreamer/libs/gst/net/gstnet.h index 28173809c27..2eebe165a90 100644 --- a/subprojects/gstreamer/libs/gst/net/gstnet.h +++ b/subprojects/gstreamer/libs/gst/net/gstnet.h @@ -27,5 +27,6 @@ #include #include #include +#include #endif /* __GST_NET_H__ */ diff --git a/subprojects/gstreamer/libs/gst/net/gsttxfeedback.c b/subprojects/gstreamer/libs/gst/net/gsttxfeedback.c new file mode 100644 index 00000000000..cbd9d3f1f6f --- /dev/null +++ b/subprojects/gstreamer/libs/gst/net/gsttxfeedback.c @@ -0,0 +1,170 @@ +/* GStreamer + * Copyright (C) <2020> Havard Graff + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "gsttxfeedback.h" + +G_DEFINE_INTERFACE (GstTxFeedback, gst_tx_feedback, 0); + +static void +gst_tx_feedback_default_init (GstTxFeedbackInterface * iface) +{ + /* default virtual functions */ + iface->tx_feedback = NULL; +} + +static void +gst_tx_feedback_send_timestamp (GstTxFeedback * parent, + guint64 buffer_id, GstClockTime ts) +{ + g_return_if_fail (GST_IS_TX_FEEDBACK (parent)); + + g_assert (GST_TX_FEEDBACK_GET_INTERFACE (parent)->tx_feedback); + + GST_TX_FEEDBACK_GET_INTERFACE (parent)->tx_feedback (parent, buffer_id, ts); +} + +static gboolean +tx_feedback_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer) +{ + GstTxFeedbackMeta *nmeta = (GstTxFeedbackMeta *) meta; + + nmeta->buffer_id = 0; + nmeta->feedback = NULL; + + return TRUE; +} + +static gboolean +tx_feedback_meta_transform (GstBuffer * transbuf, GstMeta * meta, + GstBuffer * buffer, GQuark type, gpointer data) +{ + GstTxFeedbackMeta *smeta, *dmeta; + smeta = (GstTxFeedbackMeta *) meta; + + /* we always copy no matter what transform */ + dmeta = gst_buffer_add_tx_feedback_meta (transbuf, + smeta->buffer_id, smeta->feedback); + if (!dmeta) + return FALSE; + + return TRUE; +} + +static void +tx_feedback_meta_free (GstMeta * meta, GstBuffer * buffer) +{ + GstTxFeedbackMeta *nmeta = (GstTxFeedbackMeta *) meta; + gst_clear_object (&nmeta->feedback); +} + +GType +gst_tx_feedback_meta_api_get_type (void) +{ + static GType type; + static const gchar *tags[] = { "origin", NULL }; + + if (g_once_init_enter (&type)) { + GType _type = gst_meta_api_type_register ("GstTxFeedbackMetaAPI", tags); + g_once_init_leave (&type, _type); + } + return type; +} + +const GstMetaInfo * +gst_tx_feedback_meta_get_info (void) +{ + static const GstMetaInfo *meta_info = NULL; + + if (g_once_init_enter ((GstMetaInfo **) & meta_info)) { + const GstMetaInfo *mi = gst_meta_register (GST_TX_FEEDBACK_META_API_TYPE, + "GstTxFeedbackMeta", + sizeof (GstTxFeedbackMeta), + tx_feedback_meta_init, + tx_feedback_meta_free, tx_feedback_meta_transform); + g_once_init_leave ((GstMetaInfo **) & meta_info, (GstMetaInfo *) mi); + } + return meta_info; +} + +/** + * gst_tx_feedback_meta_set_tx_time: + * @meta: a #GstTxFeedbackMeta + * @ts: a @GstClockTime with the transmit time + * + * Notifies the interface about the buffer-id being transmitted at time ts + * + * Since: 1.22 + */ +void +gst_tx_feedback_meta_set_tx_time (GstTxFeedbackMeta * meta, GstClockTime ts) +{ + gst_tx_feedback_send_timestamp (meta->feedback, meta->buffer_id, ts); +} + +/** + * gst_buffer_add_tx_feedback_meta: + * @buffer: a #GstBuffer + * @buffer_id: a #guint64 with a unique identifier for this buffer + * @feedback: a #GstTxFeedback object implementing the #GstTxFeedbackInterface + * + * Returns: (transfer none): a #GstTxFeedbackMeta connected to @buffer + * + * Since: 1.22 + */ +GstTxFeedbackMeta * +gst_buffer_add_tx_feedback_meta (GstBuffer * buffer, + guint64 buffer_id, GstTxFeedback * feedback) +{ + GstTxFeedbackMeta *meta; + + g_return_val_if_fail (GST_IS_BUFFER (buffer), NULL); + g_return_val_if_fail (GST_IS_TX_FEEDBACK (feedback), NULL); + + meta = (GstTxFeedbackMeta *) gst_buffer_add_meta (buffer, + GST_TX_FEEDBACK_META_INFO, NULL); + + meta->buffer_id = buffer_id; + meta->feedback = g_object_ref (feedback); + + return meta; +} + +/** + * gst_buffer_get_tx_feedback_meta: + * @buffer: a #GstBuffer + * + * Find the #GstTxFeedbackMeta on @buffer. + * + * Returns: (transfer none): the #GstTxFeedbackMeta or %NULL when there + * is no such metadata on @buffer. + * + * Since: 1.22 + */ +GstTxFeedbackMeta * +gst_buffer_get_tx_feedback_meta (GstBuffer * buffer) +{ + return (GstTxFeedbackMeta *) + gst_buffer_get_meta (buffer, GST_TX_FEEDBACK_META_API_TYPE); +} diff --git a/subprojects/gstreamer/libs/gst/net/gsttxfeedback.h b/subprojects/gstreamer/libs/gst/net/gsttxfeedback.h new file mode 100644 index 00000000000..e510ebdefb6 --- /dev/null +++ b/subprojects/gstreamer/libs/gst/net/gsttxfeedback.h @@ -0,0 +1,101 @@ +/* GStreamer + * Copyright (C) <2020> Havard Graff + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_TX_FEEDBACK_H__ +#define __GST_TX_FEEDBACK_H__ + +#include +#include + +G_BEGIN_DECLS + +typedef struct _GstTxFeedback GstTxFeedback; +typedef struct _GstTxFeedbackInterface GstTxFeedbackInterface; +typedef struct _GstTxFeedbackMeta GstTxFeedbackMeta; + +/** + * GstTxFeedbackInterface: + * @iface: the parent interface + * @tx_feedback: a transmit feedback + * + * The #GstTxFeedbackInterface interface. + * + * Since: 1.22 + */ +struct _GstTxFeedbackInterface { + GTypeInterface iface; + + /* virtual functions */ + void (*tx_feedback) (GstTxFeedback * feedback, + guint64 buffer_id, GstClockTime ts); +}; + +/** + * GstTxFeedbackMeta: + * @meta: the parent type + * @buffer_id: A #guint64 with an identifier for the current buffer + * + * Buffer metadata for transmission-time feedback. + * + * Since: 1.22 + */ +struct _GstTxFeedbackMeta { + GstMeta meta; + + guint64 buffer_id; + + /*< private >*/ + GstTxFeedback *feedback; +}; + +GST_NET_API +GType gst_tx_feedback_get_type (void); +#define GST_TYPE_TX_FEEDBACK (gst_tx_feedback_get_type ()) +#define GST_TX_FEEDBACK(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_TX_FEEDBACK, GstTxFeedback)) +#define GST_TX_FEEDBACK_GET_INTERFACE(obj) \ + (G_TYPE_INSTANCE_GET_INTERFACE ((obj), GST_TYPE_TX_FEEDBACK, GstTxFeedbackInterface)) +#define GST_IS_TX_FEEDBACK(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_TX_FEEDBACK)) +#define GST_TX_FEEDBACK_CAST(obj) ((GstTxFeedback *)(obj)) + +GST_NET_API +GType gst_tx_feedback_meta_api_get_type (void); +#define GST_TX_FEEDBACK_META_API_TYPE (gst_tx_feedback_meta_api_get_type()) + +GST_NET_API +const GstMetaInfo * gst_tx_feedback_meta_get_info (void); +#define GST_TX_FEEDBACK_META_INFO (gst_tx_feedback_meta_get_info()) + + +GST_NET_API +GstTxFeedbackMeta * gst_buffer_add_tx_feedback_meta (GstBuffer *buffer, + guint64 buffer_id, + GstTxFeedback *feedback); + +GST_NET_API +GstTxFeedbackMeta * gst_buffer_get_tx_feedback_meta (GstBuffer *buffer); + +GST_NET_API +void gst_tx_feedback_meta_set_tx_time (GstTxFeedbackMeta *meta, + GstClockTime ts); + +G_END_DECLS + +#endif /* __GST_TX_FEEDBACK_H__ */ diff --git a/subprojects/gstreamer/libs/gst/net/meson.build b/subprojects/gstreamer/libs/gst/net/meson.build index 17c63c8e1a4..99bac59f189 100644 --- a/subprojects/gstreamer/libs/gst/net/meson.build +++ b/subprojects/gstreamer/libs/gst/net/meson.build @@ -7,6 +7,7 @@ gst_net_sources = files( 'gstptpclock.c', 'gstntppacket.c', 'gstnetutils.c', + 'gsttxfeedback.c', ) gst_net_headers = files( @@ -18,6 +19,7 @@ gst_net_headers = files( 'gstnettimeprovider.h', 'gstnetutils.h', 'gstptpclock.h', + 'gsttxfeedback.h', 'net-prelude.h', 'net.h', ) diff --git a/subprojects/gstreamer/libs/gst/net/net.h b/subprojects/gstreamer/libs/gst/net/net.h index 4a11a94f239..6f5afd70b02 100644 --- a/subprojects/gstreamer/libs/gst/net/net.h +++ b/subprojects/gstreamer/libs/gst/net/net.h @@ -32,5 +32,6 @@ #include #include #include +#include #endif /* __GST_NET__H__ */ diff --git a/subprojects/gstreamer/meson.build b/subprojects/gstreamer/meson.build index f0d3f742f8b..bca7ab86aa0 100644 --- a/subprojects/gstreamer/meson.build +++ b/subprojects/gstreamer/meson.build @@ -76,9 +76,16 @@ if cc.get_id() == 'msvc' '/we4053', # one void operand for '?:' '/we4062', # enumerator 'identifier' in switch of enum 'enumeration' is not handled '/we4098', # 'function' : void function returning a value + ]) + + # add these warnings only if checks are enabled, + # if the checks are not built, it will leave behind some "unused" variables + if get_option('glib-checks').enabled() + msvc_args += cc.get_supported_arguments([ '/we4101', # 'identifier' : unreferenced local variable '/we4189', # 'identifier' : local variable is initialized but not referenced - ]) + ]) + endif endif add_project_arguments(msvc_args, language: 'c') elif cc.has_link_argument('-Wl,-Bsymbolic-functions') @@ -491,6 +498,11 @@ if not gst_debug add_project_arguments(['-Wno-unused'], language: 'c') endif +gst_debug_syslog = get_option('gst_debug_syslog') +if gst_debug_syslog + add_project_arguments(['-DENABLE_GST_DEBUG_SYSLOG'], language: 'c') +endif + warning_flags = [ '-Wmissing-declarations', '-Wmissing-prototypes', diff --git a/subprojects/gstreamer/meson_options.txt b/subprojects/gstreamer/meson_options.txt index 3a34bcaaa31..f201726e83f 100644 --- a/subprojects/gstreamer/meson_options.txt +++ b/subprojects/gstreamer/meson_options.txt @@ -1,4 +1,5 @@ option('gst_debug', type : 'boolean', value : true) +option('gst_debug_syslog', type : 'boolean', value : false) option('gst_parse', type : 'boolean', value : true, description: 'Enable pipeline string parser') option('registry', type : 'boolean', value : true) diff --git a/subprojects/gstreamer/plugins/elements/gstfilesrc.c b/subprojects/gstreamer/plugins/elements/gstfilesrc.c index 5dfe014169b..bfd3b63f253 100644 --- a/subprojects/gstreamer/plugins/elements/gstfilesrc.c +++ b/subprojects/gstreamer/plugins/elements/gstfilesrc.c @@ -110,11 +110,13 @@ enum }; #define DEFAULT_BLOCKSIZE 4*1024 +#define DEFAULT_LOOP 0 enum { PROP_0, - PROP_LOCATION + PROP_LOCATION, + PROP_LOOP }; static void gst_file_src_finalize (GObject * object); @@ -163,6 +165,11 @@ gst_file_src_class_init (GstFileSrcClass * klass) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_READY)); + g_object_class_install_property (gobject_class, PROP_LOOP, + g_param_spec_int ("loop", "Loop", + "How many times to loop when reaching EOF", 0, G_MAXINT, DEFAULT_LOOP, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + gobject_class->finalize = gst_file_src_finalize; gst_element_class_set_static_metadata (gstelement_class, @@ -191,6 +198,9 @@ gst_file_src_init (GstFileSrc * src) src->fd = 0; src->uri = NULL; + src->loop = 0; + src->filesize = G_MAXUINT64; + src->is_regular = FALSE; gst_base_src_set_blocksize (GST_BASE_SRC (src), DEFAULT_BLOCKSIZE); @@ -270,6 +280,9 @@ gst_file_src_set_property (GObject * object, guint prop_id, case PROP_LOCATION: gst_file_src_set_location (src, g_value_get_string (value), NULL); break; + case PROP_LOOP: + src->loop = g_value_get_int (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -290,6 +303,9 @@ gst_file_src_get_property (GObject * object, guint prop_id, GValue * value, case PROP_LOCATION: g_value_set_string (value, src->filename); break; + case PROP_LOOP: + g_value_set_int (value, src->loop); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -318,6 +334,8 @@ gst_file_src_fill (GstBaseSrc * basesrc, guint64 offset, guint length, src = GST_FILE_SRC_CAST (basesrc); + offset = offset % src->filesize; + if (G_UNLIKELY (offset != -1 && src->read_position != offset)) { off_t res; @@ -440,6 +458,9 @@ gst_file_src_get_size (GstBaseSrc * basesrc, guint64 * size) } #endif + src->filesize = *size; + *size = src->filesize * (src->loop + 1); + return TRUE; /* ERROR */ diff --git a/subprojects/gstreamer/plugins/elements/gstfilesrc.h b/subprojects/gstreamer/plugins/elements/gstfilesrc.h index e73cfc3d5ea..b831eac2adc 100644 --- a/subprojects/gstreamer/plugins/elements/gstfilesrc.h +++ b/subprojects/gstreamer/plugins/elements/gstfilesrc.h @@ -63,6 +63,9 @@ struct _GstFileSrc { gboolean seekable; /* whether the file is seekable */ gboolean is_regular; /* whether it's a (symlink to a) regular file */ + + guint64 filesize; /* file size */ + gint loop; /* number of loops before EOS */ }; struct _GstFileSrcClass { diff --git a/subprojects/gstreamer/plugins/elements/gstfunnel.c b/subprojects/gstreamer/plugins/elements/gstfunnel.c index b09dbf18d11..4575cc63996 100644 --- a/subprojects/gstreamer/plugins/elements/gstfunnel.c +++ b/subprojects/gstreamer/plugins/elements/gstfunnel.c @@ -394,23 +394,8 @@ gst_funnel_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) GST_OBJECT_LOCK (funnel); fpad->got_eos = FALSE; GST_OBJECT_UNLOCK (funnel); - } - - if (forward && GST_EVENT_IS_SERIALIZED (event)) { - /* If no data is coming and we receive serialized event, need to forward all sticky events. - * Otherwise downstream has an inconsistent set of sticky events when - * handling the new event. */ - if (!unlock) { - unlock = TRUE; - GST_PAD_STREAM_LOCK (funnel->srcpad); - } - - if ((funnel->last_sinkpad == NULL) || (funnel->forward_sticky_events - && (funnel->last_sinkpad != pad))) { - gst_object_replace ((GstObject **) & funnel->last_sinkpad, - GST_OBJECT (pad)); - gst_pad_sticky_events_foreach (pad, forward_events, funnel->srcpad); - } + } else if (pad != funnel->last_sinkpad) { + forward = FALSE; } if (forward) diff --git a/subprojects/gstreamer/plugins/tracers/gstleaks.c b/subprojects/gstreamer/plugins/tracers/gstleaks.c index b586283496c..97a117f8512 100644 --- a/subprojects/gstreamer/plugins/tracers/gstleaks.c +++ b/subprojects/gstreamer/plugins/tracers/gstleaks.c @@ -355,6 +355,20 @@ object_log_free (ObjectLog * obj) g_free (obj); } +static guint +object_log_hash (const ObjectLog * obj) +{ + return (guint) obj->object + (guint) obj->type_qname; +} + +static gboolean +object_log_compare (const ObjectLog * a, const ObjectLog * b) +{ + if ((a->object == b->object) && (a->type_qname == b->type_qname)) + return TRUE; + return FALSE; +} + static void handle_object_destroyed (GstLeaksTracer * self, gpointer object, ObjectKind kind) @@ -1048,9 +1062,9 @@ gst_leaks_tracer_activity_start_tracking (GstLeaksTracer * self) return; } - self->added = g_hash_table_new_full (NULL, NULL, + self->added = g_hash_table_new_full (object_log_hash, object_log_compare, (GDestroyNotify) object_log_free, NULL); - self->removed = g_hash_table_new_full (NULL, NULL, + self->removed = g_hash_table_new_full (object_log_hash, object_log_compare, (GDestroyNotify) object_log_free, NULL); GST_OBJECT_UNLOCK (self); } @@ -1091,26 +1105,51 @@ process_checkpoint (GstTracerRecord * record, const gchar * record_type, } } + +static GHashTable * +_copy_and_remove (GHashTable * a, GHashTable * b) +{ + GHashTable *copy = g_hash_table_new (NULL, NULL); + GHashTableIter iter; + gpointer o; + + g_hash_table_iter_init (&iter, a); + while (g_hash_table_iter_next (&iter, &o, NULL)) { + if (!g_hash_table_contains (b, o)) + g_hash_table_add (copy, o); + } + + return copy; +} + static GstStructure * gst_leaks_tracer_activity_get_checkpoint (GstLeaksTracer * self) { GValue added = G_VALUE_INIT; GValue removed = G_VALUE_INIT; + GValue alive = G_VALUE_INIT; GstStructure *s = gst_structure_new_empty ("activity-checkpoint"); g_value_init (&added, GST_TYPE_LIST); g_value_init (&removed, GST_TYPE_LIST); + g_value_init (&alive, GST_TYPE_LIST); GST_OBJECT_LOCK (self); + + GHashTable *copy = _copy_and_remove (self->added, self->removed); + process_checkpoint (tr_added, "objects-created", self->added, &added); process_checkpoint (tr_removed, "objects-removed", self->removed, &removed); + process_checkpoint (NULL, "objects-created", copy, &alive); g_hash_table_remove_all (self->added); g_hash_table_remove_all (self->removed); + g_hash_table_destroy (copy); GST_OBJECT_UNLOCK (self); gst_structure_take_value (s, "objects-created-list", &added); gst_structure_take_value (s, "objects-removed-list", &removed); + gst_structure_take_value (s, "objects-alive-list", &alive); return s; } diff --git a/subprojects/gstreamer/tests/check/elements/funnel.c b/subprojects/gstreamer/tests/check/elements/funnel.c index d88628e4316..b9efa92a3bb 100644 --- a/subprojects/gstreamer/tests/check/elements/funnel.c +++ b/subprojects/gstreamer/tests/check/elements/funnel.c @@ -108,7 +108,8 @@ static gint bufcount = 0; static gint alloccount = 0; static GstFlowReturn -chain_ok (GstPad * pad, GstObject * parent, GstBuffer * buffer) +chain_ok (G_GNUC_UNUSED GstPad * pad, G_GNUC_UNUSED GstObject * parent, + GstBuffer * buffer) { bufcount++; @@ -244,135 +245,6 @@ GST_START_TEST (test_funnel_eos) GST_END_TEST; -guint nb_stream_start_event = 0; -guint nb_caps_event = 0; -guint nb_segment_event = 0; -guint nb_gap_event = 0; - -static GstPadProbeReturn -event_counter (GstObject * pad, GstPadProbeInfo * info, gpointer user_data) -{ - GstEvent *event = GST_PAD_PROBE_INFO_EVENT (info); - - fail_unless (event != NULL); - fail_unless (GST_IS_EVENT (event)); - - switch (GST_EVENT_TYPE (event)) { - case GST_EVENT_STREAM_START: - ++nb_stream_start_event; - break; - case GST_EVENT_CAPS: - ++nb_caps_event; - break; - case GST_EVENT_SEGMENT: - ++nb_segment_event; - break; - case GST_EVENT_GAP: - ++nb_gap_event; - break; - default: - break; - } - - return GST_PAD_PROBE_OK; -} - -/* - * Push GAP events into funnel to forward sticky events. - * Funnel element should also treat GAP events likes buffers. - * For example, funnel can be used for internal subtitle with streamiddemux. - * +--------------------------------------------------------------------------+ - * | playbin +--------------------------------+ | - * | +--------------+ +----------------+ | +------------+ playsink | | - * | | uridecodebin | | input-selector | | | video-sink | | | - * | | | +----------------+ | +------------+ | | - * | | | | | | - * | | | +----------------+ | +------------+ | | - * | | | | input-selector | | | audio-sink | | | - * | | | +----------------+ | +------------+ | | - * | | | | | | - * | | | +----------------+ | +---------------+ +----------+ | | - * | | | | funnel | | | streamiddemux | | appsink0 | | | - * | +--------------+ +----------------+ | +---------------+ +----------+ | | - * | | +----------+ | | - * | | | appsinkn | | | - * | | +----------+ | | - * | +--------------------------------+ | - * +--------------------------------------------------------------------------+ - * If no data was received in funnel and then sticky events can be pending continuously. - * And streamiddemux only receive gap events continuously. - * Thus, pipeline can not be constructed completely. - * For support it, need to handle GAP events likes buffers. - */ -GST_START_TEST (test_funnel_gap_event) -{ - struct TestData td; - guint probe = 0; - - setup_test_objects (&td, chain_ok); - - nb_stream_start_event = 0; - nb_caps_event = 0; - nb_segment_event = 0; - nb_gap_event = 0; - bufcount = 0; - - probe = gst_pad_add_probe (td.mysink, GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, - (GstPadProbeCallback) event_counter, NULL, NULL); - - /* push a gap event to srcpad1 to push sticky events */ - fail_unless (gst_pad_push_event (td.mysrc1, gst_event_new_gap (0, - GST_SECOND))); - - fail_unless (nb_stream_start_event == 1); - fail_unless (nb_caps_event == 1); - fail_unless (nb_segment_event == 1); - fail_unless (nb_gap_event == 1); - - /* push a gap event to srcpad2 to push sticky events */ - fail_unless (gst_pad_push_event (td.mysrc2, gst_event_new_gap (0, - GST_SECOND))); - - fail_unless (nb_stream_start_event == 2); - fail_unless (nb_caps_event == 2); - fail_unless (nb_segment_event == 2); - fail_unless (nb_gap_event == 2); - - /* push a gap event to srcpad2 */ - fail_unless (gst_pad_push_event (td.mysrc2, gst_event_new_gap (0, - GST_SECOND))); - - fail_unless (nb_stream_start_event == 2); - fail_unless (nb_caps_event == 2); - fail_unless (nb_segment_event == 2); - fail_unless (nb_gap_event == 3); - - /* push a gap event to srcpad1 */ - fail_unless (gst_pad_push_event (td.mysrc1, gst_event_new_gap (0, - GST_SECOND))); - - fail_unless (nb_stream_start_event == 3); - fail_unless (nb_caps_event == 3); - fail_unless (nb_segment_event == 3); - fail_unless (nb_gap_event == 4); - - /* push buffer */ - fail_unless (gst_pad_push (td.mysrc1, gst_buffer_new ()) == GST_FLOW_OK); - fail_unless (gst_pad_push (td.mysrc2, gst_buffer_new ()) == GST_FLOW_OK); - - fail_unless (nb_stream_start_event == 4); - fail_unless (nb_caps_event == 4); - fail_unless (nb_segment_event == 4); - fail_unless (nb_gap_event == 4); - fail_unless (bufcount == 2); - - gst_pad_remove_probe (td.mysink, probe); - - release_test_objects (&td); -} - -GST_END_TEST; - GST_START_TEST (test_funnel_stress) { GstHarness *h0 = gst_harness_new_with_padnames ("funnel", "sink_0", "src"); @@ -407,6 +279,194 @@ GST_START_TEST (test_funnel_stress) GST_END_TEST; +GST_START_TEST (test_funnel_event_handling) +{ + GstHarness *h, *h0, *h1; + GstEvent *event; + GstCaps *mycaps0, *mycaps1, *caps; + + h = gst_harness_new_with_padnames ("funnel", NULL, "src"); + + /* request a sinkpad, with some caps */ + h0 = gst_harness_new_with_element (h->element, "sink_0", NULL); + mycaps0 = gst_caps_new_empty_simple ("mycaps0"); + gst_harness_set_src_caps (h0, gst_caps_ref (mycaps0)); + + /* request a second sinkpad, also with caps */ + h1 = gst_harness_new_with_element (h->element, "sink_1", NULL); + mycaps1 = gst_caps_new_empty_simple ("mycaps1"); + gst_harness_set_src_caps (h1, gst_caps_ref (mycaps1)); + + /* push a buffer on the first pad */ + fail_unless_equals_int (GST_FLOW_OK, + gst_harness_push (h0, gst_buffer_new ())); + gst_buffer_unref (gst_harness_pull (h)); + + /* verify stream-start */ + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_TYPE (event), GST_EVENT_STREAM_START); + gst_event_unref (event); + + /* verify caps "mycaps0" */ + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_TYPE (event), GST_EVENT_CAPS); + gst_event_parse_caps (event, &caps); + gst_check_caps_equal (mycaps0, caps); + gst_caps_unref (caps); + gst_event_unref (event); + + /* verify segment */ + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_TYPE (event), GST_EVENT_SEGMENT); + gst_event_unref (event); + + /* verify a custom event pushed on second pad does not make it through, + since active pad is the first pad */ + fail_unless (gst_harness_push_event (h1, + gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM, + gst_structure_new_empty ("test")))); + fail_if (gst_harness_try_pull_event (h)); + + /* but same event on the first pad will make it through */ + fail_unless (gst_harness_push_event (h0, + gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM, + gst_structure_new_empty ("test")))); + gst_event_unref (gst_harness_pull_event (h)); + + /* push a buffer on the second pad */ + fail_unless_equals_int (GST_FLOW_OK, + gst_harness_push (h1, gst_buffer_new ())); + gst_buffer_unref (gst_harness_pull (h)); + + /* verify stream-start */ + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_TYPE (event), GST_EVENT_STREAM_START); + gst_event_unref (event); + + /* verify caps "mycaps1" */ + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_TYPE (event), GST_EVENT_CAPS); + gst_event_parse_caps (event, &caps); + gst_check_caps_equal (mycaps1, caps); + gst_caps_unref (caps); + gst_event_unref (event); + + /* verify segment */ + event = gst_harness_pull_event (h); + fail_unless_equals_int (GST_EVENT_TYPE (event), GST_EVENT_SEGMENT); + gst_event_unref (event); + + /* verify a custom event pushed on first pad does not make it through, + since active pad is the second pad */ + fail_unless (gst_harness_push_event (h0, + gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM, + gst_structure_new_empty ("test")))); + fail_if (gst_harness_try_pull_event (h)); + + /* but same event on the second pad will make it through */ + fail_unless (gst_harness_push_event (h1, + gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM, + gst_structure_new_empty ("test")))); + gst_event_unref (gst_harness_pull_event (h)); + + gst_harness_teardown (h); + gst_harness_teardown (h0); + gst_harness_teardown (h1); +} + +GST_END_TEST; + +GST_START_TEST (test_funnel_custom_sticky) +{ + GstHarness *h, *h0, *h1; + GstEvent *event; + const GstStructure *s; + const gchar *value = NULL; + + h = gst_harness_new_with_padnames ("funnel", NULL, "src"); + + /* request a sinkpad, with some caps */ + h0 = gst_harness_new_with_element (h->element, "sink_0", NULL); + gst_harness_set_src_caps_str (h0, "mycaps0"); + + /* request a second sinkpad, also with caps */ + h1 = gst_harness_new_with_element (h->element, "sink_1", NULL); + gst_harness_set_src_caps_str (h1, "mycaps1"); + + while ((event = gst_harness_try_pull_event (h))) + gst_event_unref (event); + + fail_unless (gst_harness_push_event (h0, + gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM_STICKY, + gst_structure_new ("test", "key", G_TYPE_STRING, "value0", + NULL)))); + + fail_unless (gst_harness_push_event (h1, + gst_event_new_custom (GST_EVENT_CUSTOM_DOWNSTREAM_STICKY, + gst_structure_new ("test", "key", G_TYPE_STRING, "value1", + NULL)))); + + /* Send a buffer through first pad, expect the event to be the first one */ + fail_unless_equals_int (GST_FLOW_OK, + gst_harness_push (h0, gst_buffer_new ())); + for (;;) { + event = gst_harness_pull_event (h); + if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_DOWNSTREAM_STICKY) + break; + gst_event_unref (event); + } + s = gst_event_get_structure (event); + fail_unless (s); + fail_unless (gst_structure_has_name (s, "test")); + value = gst_structure_get_string (s, "key"); + fail_unless_equals_string (value, "value0"); + gst_event_unref (event); + gst_buffer_unref (gst_harness_pull (h)); + + /* Send a buffer through second pad, expect the event to be the second one + */ + fail_unless_equals_int (GST_FLOW_OK, + gst_harness_push (h1, gst_buffer_new ())); + for (;;) { + event = gst_harness_pull_event (h); + if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_DOWNSTREAM_STICKY) + break; + gst_event_unref (event); + } + s = gst_event_get_structure (event); + fail_unless (s); + fail_unless (gst_structure_has_name (s, "test")); + value = gst_structure_get_string (s, "key"); + fail_unless_equals_string (value, "value1"); + gst_event_unref (event); + gst_buffer_unref (gst_harness_pull (h)); + + /* Send a buffer through first pad, expect the event to again be the first + * one + */ + fail_unless_equals_int (GST_FLOW_OK, + gst_harness_push (h0, gst_buffer_new ())); + for (;;) { + event = gst_harness_pull_event (h); + if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_DOWNSTREAM_STICKY) + break; + gst_event_unref (event); + } + s = gst_event_get_structure (event); + fail_unless (s); + fail_unless (gst_structure_has_name (s, "test")); + value = gst_structure_get_string (s, "key"); + fail_unless_equals_string (value, "value0"); + gst_event_unref (event); + gst_buffer_unref (gst_harness_pull (h)); + + gst_harness_teardown (h); + gst_harness_teardown (h0); + gst_harness_teardown (h1); +} + +GST_END_TEST; + static Suite * funnel_suite (void) @@ -417,8 +477,9 @@ funnel_suite (void) tc_chain = tcase_create ("funnel simple"); tcase_add_test (tc_chain, test_funnel_simple); tcase_add_test (tc_chain, test_funnel_eos); - tcase_add_test (tc_chain, test_funnel_gap_event); tcase_add_test (tc_chain, test_funnel_stress); + tcase_add_test (tc_chain, test_funnel_event_handling); + tcase_add_test (tc_chain, test_funnel_custom_sticky); suite_add_tcase (s, tc_chain); return s; diff --git a/subprojects/gstreamer/tests/check/gst/gstbin.c b/subprojects/gstreamer/tests/check/gst/gstbin.c index 88ff44db0c3..0f50d283ecb 100644 --- a/subprojects/gstreamer/tests/check/gst/gstbin.c +++ b/subprojects/gstreamer/tests/check/gst/gstbin.c @@ -25,6 +25,7 @@ #include #include +#include static void pop_async_done (GstBus * bus, gboolean * had_latency) @@ -1988,6 +1989,57 @@ GST_START_TEST (test_suppressed_flags_when_removing) GST_END_TEST; +GST_START_TEST (test_locked_state_sets_base_time_on_children) +{ + GstElement *pipeline, *bin, *identity; + GstClock *testclock; + GstStateChangeReturn state_res; + GstState current, pending; + + /* create the pipeline, and have it use the testclock */ + pipeline = gst_pipeline_new (NULL); + testclock = gst_test_clock_new (); + gst_pipeline_use_clock (GST_PIPELINE (pipeline), testclock); + + /* create a bin, and add an identity element to it */ + bin = gst_bin_new (NULL); + identity = gst_element_factory_make ("identity", NULL); + gst_bin_add (GST_BIN (bin), identity); + + /* add the bin to the pipeline and set locked-state, + protecting the bin from the pipeline state-changes */ + gst_bin_add (GST_BIN (pipeline), bin); + gst_element_set_locked_state (bin, TRUE); + + /* advance the clock, and set to playing, making this our base-time */ + gst_test_clock_set_time (GST_TEST_CLOCK (testclock), 123456789); + state_res = gst_element_set_state (pipeline, GST_STATE_PLAYING); + fail_unless_equals_int (GST_STATE_CHANGE_SUCCESS, state_res); + fail_unless_equals_int64 (123456789, gst_element_get_base_time (pipeline)); + + /* verify that neither the bin nor the identity-element got affected by + the state-change */ + state_res = gst_element_get_state (GST_ELEMENT (bin), + ¤t, &pending, GST_CLOCK_TIME_NONE); + fail_unless_equals_int (GST_STATE_CHANGE_SUCCESS, state_res); + fail_unless_equals_int (GST_STATE_NULL, current); + fail_unless_equals_int (GST_STATE_VOID_PENDING, pending); + + state_res = gst_element_get_state (GST_ELEMENT (identity), + ¤t, &pending, GST_CLOCK_TIME_NONE); + fail_unless_equals_int (GST_STATE_CHANGE_SUCCESS, state_res); + fail_unless_equals_int (GST_STATE_NULL, current); + fail_unless_equals_int (GST_STATE_VOID_PENDING, pending); + + /* and check that base-time got distributed to everyone */ + fail_unless_equals_int64 (123456789, gst_element_get_base_time (bin)); + fail_unless_equals_int64 (123456789, gst_element_get_base_time (identity)); + + gst_object_unref (pipeline); +} + +GST_END_TEST; + static Suite * gst_bin_suite (void) { @@ -2023,6 +2075,7 @@ gst_bin_suite (void) tcase_add_test (tc_chain, test_deep_added_removed); tcase_add_test (tc_chain, test_suppressed_flags); tcase_add_test (tc_chain, test_suppressed_flags_when_removing); + tcase_skip_broken_test (tc_chain, test_locked_state_sets_base_time_on_children); /* fails on OSX build bot for some reason, and is a bit silly anyway */ if (0) diff --git a/subprojects/gstreamer/tests/check/gst/gstelement.c b/subprojects/gstreamer/tests/check/gst/gstelement.c index a80aa45e675..de5108ae988 100644 --- a/subprojects/gstreamer/tests/check/gst/gstelement.c +++ b/subprojects/gstreamer/tests/check/gst/gstelement.c @@ -962,6 +962,106 @@ GST_START_TEST (test_foreach_pad) GST_END_TEST; +typedef struct +{ + GstPad *srcpads[1000]; +} ReleaseDisposeCtx; + +static void +_release_request_pad (GstPad * pad) +{ + GstElement *parent; + if (pad == NULL) + return; + parent = GST_ELEMENT (gst_pad_get_parent (pad)); + if (parent) { + gst_element_release_request_pad (parent, pad); + gst_object_unref (parent); + } + gst_object_unref (pad); +} + +static gpointer +_release_pad_func (gpointer user_data) +{ + ReleaseDisposeCtx *ctx = user_data; + for (guint i = 0; i < G_N_ELEMENTS (ctx->srcpads); i++) + _release_request_pad (ctx->srcpads[i]); + return NULL; +} + +GST_START_TEST (test_release_pads_during_dispose) +{ + GstElement *pipeline = gst_pipeline_new (NULL); + GstTestElement3 *te3 = g_object_new (gst_test_element3_get_type (), NULL); + GstElement *element = GST_ELEMENT_CAST (te3); + ReleaseDisposeCtx ctx; + GThread *query_thread; + + gst_bin_add (GST_BIN (pipeline), element); + + /* request lots of pads */ + for (guint i = 0; i < G_N_ELEMENTS (ctx.srcpads); i++) { + ctx.srcpads[i] = gst_element_get_request_pad (element, "src_%d"); + } + + /* start a thread to start releasing those pads */ + query_thread = g_thread_new (NULL, _release_pad_func, &ctx); + + /* while at the same time, shutting down the pipeline */ + gst_element_set_state (pipeline, GST_STATE_NULL); + gst_object_unref (pipeline); + + g_thread_join (query_thread); +} + +GST_END_TEST; + +static void +add_srcpad_deadlock_loop (void) +{ +} + +static gboolean +add_srcpad_deadlock_activate_mode (GstPad * pad, GstObject * parent, + GstPadMode mode, gboolean active) +{ + if (active) + return gst_pad_start_task (pad, (GstTaskFunction) add_srcpad_deadlock_loop, + NULL, NULL); + + return gst_pad_stop_task (pad); +} + +GST_START_TEST (test_add_srcpad_deadlock) +{ + GstTestElement3 *te3 = g_object_new (gst_test_element3_get_type (), NULL); + GstPadTemplate *pad_template; + GstPad *srcpad; + + pad_template = + gst_element_class_get_pad_template (GST_ELEMENT_GET_CLASS (te3), + "src_%u"); + g_return_if_fail (pad_template != NULL); + + srcpad = gst_pad_new_from_template (pad_template, "src"); + g_return_if_fail (srcpad != NULL); + gst_pad_set_activatemode_function (srcpad, add_srcpad_deadlock_activate_mode); + + gst_element_set_state (GST_ELEMENT (te3), GST_STATE_PLAYING); + + /* gst_element_add_pad should activate the pad, causing activate_mode and + gst_pad_start_task to be called */ + gst_element_add_pad (GST_ELEMENT (te3), srcpad); + + gst_element_set_state (GST_ELEMENT (te3), GST_STATE_NULL); + + gst_object_unref (te3); +} + +GST_END_TEST; + + static Suite * gst_element_suite (void) { @@ -980,6 +1080,8 @@ gst_element_suite (void) tcase_add_test (tc_chain, test_request_pad_templates); tcase_add_test (tc_chain, test_forbidden_pad_template_names); tcase_add_test (tc_chain, test_foreach_pad); + tcase_add_test (tc_chain, test_release_pads_during_dispose); + tcase_add_test (tc_chain, test_add_srcpad_deadlock); return s; } diff --git a/subprojects/gstreamer/tests/check/gst/gstinfo.c b/subprojects/gstreamer/tests/check/gst/gstinfo.c index 8615e4802a0..43120ec8027 100644 --- a/subprojects/gstreamer/tests/check/gst/gstinfo.c +++ b/subprojects/gstreamer/tests/check/gst/gstinfo.c @@ -558,6 +558,74 @@ GST_START_TEST (info_set_and_reset_string) GST_END_TEST; + +typedef struct +{ + gboolean thread_running; + gpointer user_data; +} StressTestData; + +static gpointer +debug_pad_name_func (gpointer user_data) +{ + StressTestData *data = user_data; + GstPad *pad = data->user_data; + + while (data->thread_running) { + gchar *name = g_strdup_printf ("%s:%s", GST_DEBUG_PAD_NAME (pad)); + fail_unless (name != NULL); + g_free (name); + } + + return NULL; +} + +static gpointer +unparent_func (gpointer user_data) +{ + StressTestData *data = user_data; + GstObject *obj = data->user_data; + + while (data->thread_running) { + GstObject *parent = gst_object_get_parent (obj); + gst_object_unparent (obj); + g_thread_yield (); + gst_object_set_parent (obj, parent); + gst_object_unref (parent); + } + + return NULL; +} + +GST_START_TEST(gst_debug_pad_name_stress) +{ + GThread *pad_name_thread, *unparent_thread; + GstPad *pad = gst_pad_new ("testpad", GST_PAD_UNKNOWN); + GstElement *element = gst_element_factory_make ("identity", NULL); + StressTestData data = { TRUE, pad }; + + g_object_ref (G_OBJECT (pad)); + gst_object_set_parent (GST_OBJECT (pad), GST_OBJECT (element)); + + pad_name_thread = g_thread_new ( + "pad_name_thread", (GThreadFunc) debug_pad_name_func, &data); + unparent_thread = g_thread_new ( + "gst_object_unparent_thread", (GThreadFunc) unparent_func, &data); + + /* test duration */ + g_usleep (G_USEC_PER_SEC / 10); + + data.thread_running = FALSE; + g_thread_join (pad_name_thread); + g_thread_join (unparent_thread); + + gst_object_unparent (GST_OBJECT (pad)); + gst_object_unref (pad); + gst_object_unref (element); +} + +GST_END_TEST; + static Suite * gst_info_suite (void) { @@ -583,6 +651,12 @@ gst_info_suite (void) tcase_add_test (tc_chain, info_set_and_reset_string); #endif +#if defined (__GNUC__) + tcase_add_test (tc_chain, gst_debug_pad_name_stress); +#else + tcase_skip_broken_test (tc_chain, gst_debug_pad_name_stress); +#endif + return s; } diff --git a/subprojects/gstreamer/tests/check/gst/gstvalue.c b/subprojects/gstreamer/tests/check/gst/gstvalue.c index b1d2548584c..c9f055a784c 100644 --- a/subprojects/gstreamer/tests/check/gst/gstvalue.c +++ b/subprojects/gstreamer/tests/check/gst/gstvalue.c @@ -450,7 +450,7 @@ GST_END_TEST; GST_START_TEST (test_serialize_flags_invalid) { GValue value = { 0 }; - gchar *string; + gchar *string = NULL; g_value_init (&value, GST_TYPE_SEEK_FLAGS); diff --git a/subprojects/gstreamer/tests/check/gst/struct_x86_64.h b/subprojects/gstreamer/tests/check/gst/struct_x86_64.h index 2708df01cc7..20cfca55684 100644 --- a/subprojects/gstreamer/tests/check/gst/struct_x86_64.h +++ b/subprojects/gstreamer/tests/check/gst/struct_x86_64.h @@ -55,7 +55,7 @@ static GstCheckABIStruct list[] = { {"GstSystemClockClass", sizeof (GstSystemClockClass), 296}, {"GstSystemClock", sizeof (GstSystemClock), 168}, {"GstTagList", sizeof (GstTagList), 64,}, - {"GstTaskClass", sizeof (GstTaskClass), 224}, + {"GstTaskClass", sizeof (GstTaskClass), 232}, {"GstTask", sizeof (GstTask), 200}, {"GstTaskPoolClass", sizeof (GstTaskPoolClass), 248}, {"GstTaskPool", sizeof (GstTaskPool), 128}, diff --git a/subprojects/gstreamer/tests/check/gst/struct_x86_64w.h b/subprojects/gstreamer/tests/check/gst/struct_x86_64w.h index 00c640589e1..51b35d2a135 100644 --- a/subprojects/gstreamer/tests/check/gst/struct_x86_64w.h +++ b/subprojects/gstreamer/tests/check/gst/struct_x86_64w.h @@ -55,7 +55,7 @@ static GstCheckABIStruct list[] = { {"GstSystemClockClass", sizeof (GstSystemClockClass), 296}, {"GstSystemClock", sizeof (GstSystemClock), 168}, {"GstTagList", sizeof (GstTagList), 64,}, - {"GstTaskClass", sizeof (GstTaskClass), 224}, + {"GstTaskClass", sizeof (GstTaskClass), 232}, {"GstTask", sizeof (GstTask), 200}, {"GstTaskPoolClass", sizeof (GstTaskPoolClass), 248}, {"GstTaskPool", sizeof (GstTaskPool), 128}, diff --git a/subprojects/gstreamer/tests/check/gstreamer.supp b/subprojects/gstreamer/tests/check/gstreamer.supp index e69c0cfb7be..586efeadcd4 100644 --- a/subprojects/gstreamer/tests/check/gstreamer.supp +++ b/subprojects/gstreamer/tests/check/gstreamer.supp @@ -4035,6 +4035,15 @@ fun:gst_debug_add_log_function } +{ + Leak of debug function list + Memcheck:Leak + fun:*alloc + ... + fun:g_slist_copy* + fun:gst_debug_remove_with_compare_func +} + { Leak of debug function list item Memcheck:Leak diff --git a/subprojects/gstreamer/tests/check/libs/baseidlesrc.c b/subprojects/gstreamer/tests/check/libs/baseidlesrc.c new file mode 100644 index 00000000000..7d7aa1a4a39 --- /dev/null +++ b/subprojects/gstreamer/tests/check/libs/baseidlesrc.c @@ -0,0 +1,112 @@ +/* GStreamer + * + * some unit tests for GstBaseIdleSrc + * + * Copyright (C) 2023 Havard Graff + * 2023 Camilo Celis Guzman + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include +#include +#include +#include +#include + +typedef GstBaseIdleSrc TestIdleSrc; +typedef GstBaseIdleSrcClass TestIdleSrcClass; + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +static GType test_idle_src_get_type (void); + +G_DEFINE_TYPE (TestIdleSrc, test_idle_src, GST_TYPE_BASE_IDLE_SRC); + +static void +test_idle_src_init (TestIdleSrc * src) +{ +} + +static void +test_idle_src_class_init (TestIdleSrcClass * klass) +{ + gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass), + &src_template); +} + +GST_START_TEST (baseidlesrc_up_and_down) +{ + GstElement *src; + GstHarness *h; + + src = g_object_new (test_idle_src_get_type (), NULL); + + h = gst_harness_new_with_element (GST_ELEMENT (src), NULL, "src"); + + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (baseidlesrc_submit_buffer) +{ + GstElement *src; + GstHarness *h; + GstBaseIdleSrc *base_src; + GstBuffer *buf; + guint i; + + src = g_object_new (test_idle_src_get_type (), NULL); + base_src = GST_BASE_IDLE_SRC (src); + + h = gst_harness_new_with_element (GST_ELEMENT (src), NULL, "src"); + + gst_harness_set_sink_caps_str (h, "foo/bar"); + gst_harness_play (h); + + for (i = 0; i < 5; i++) { + fail_unless_equals_int (GST_FLOW_OK, + gst_base_idle_src_alloc_buffer (base_src, 100, &buf)); + gst_base_idle_src_submit_buffer (base_src, buf); + gst_buffer_unref (gst_harness_pull (h)); + } + + gst_harness_teardown (h); +} + +GST_END_TEST; + +static Suite * +baseidlesrc_suite (void) +{ + Suite *s = suite_create ("GstBaseIdleSrc"); + TCase *tc = tcase_create ("general"); + + suite_add_tcase (s, tc); + tcase_add_test (tc, baseidlesrc_up_and_down); + tcase_add_test (tc, baseidlesrc_submit_buffer); + + return s; +} + +GST_CHECK_MAIN (baseidlesrc); diff --git a/subprojects/gstreamer/tests/check/libs/basesrc.c b/subprojects/gstreamer/tests/check/libs/basesrc.c index 253d69ab64b..df863424434 100644 --- a/subprojects/gstreamer/tests/check/libs/basesrc.c +++ b/subprojects/gstreamer/tests/check/libs/basesrc.c @@ -276,7 +276,7 @@ GST_END_TEST; * - makes sure source doesn't send EOS event when reaching the max. * number of buffers configured in pull-mode * - make sure source doesn't send EOS event either when being shut down - * (PAUSED => READY state change) after EOSing in pull mode + * (PAUSED => READY state change) after EOSing in pull mode */ GST_START_TEST (basesrc_eos_events_pull) { diff --git a/subprojects/gstreamer/tests/check/libs/gstharness.c b/subprojects/gstreamer/tests/check/libs/gstharness.c index 7999b4bb944..ac26f5cf577 100644 --- a/subprojects/gstreamer/tests/check/libs/gstharness.c +++ b/subprojects/gstreamer/tests/check/libs/gstharness.c @@ -1,7 +1,7 @@ /* * Tests and examples of GstHarness * - * Copyright (C) 2015 Havard Graff + * Copyright (C) 2015-2021 Havard Graff * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -22,6 +22,8 @@ #include "config.h" #endif +#include + #include #include @@ -293,6 +295,103 @@ GST_START_TEST (test_get_all_data) GST_END_TEST; +static GstFlowReturn +gst_harness_dummy_identity_chain_list (GstPad * pad, GstObject * parent, + GstBufferList * buffer_list) +{ + GstBaseTransform *trans = GST_BASE_TRANSFORM_CAST (parent); + return gst_pad_push_list (trans->srcpad, buffer_list); +} + +GST_START_TEST (test_src_harness_buflist) +{ + GstBaseTransform *trans; + GstBufferList *list, *push_list; + GstHarness *h; + GstBuffer *buffer; + + h = gst_harness_new ("identity"); + gst_harness_set_src_caps_str (h, "mycaps"); + + /* Make GstIdentity element forward buffer lists as is instead of destructing + them into individual buffers */ + trans = GST_BASE_TRANSFORM_CAST (gst_harness_find_element (h, "identity")); + gst_pad_set_chain_list_function (trans->sinkpad, + GST_DEBUG_FUNCPTR (gst_harness_dummy_identity_chain_list)); + + /* Push and pull list */ + push_list = gst_buffer_list_new_sized (2); + for (int i = 0; i < 2; ++i) { + buffer = gst_buffer_new_allocate (NULL, i*17, NULL); + gst_buffer_list_add (push_list, buffer); + } + + gst_harness_push_list (h, push_list); + + /* Verify that identity outputs a buffer by pulling and unreffing */ + list = gst_harness_pull_list (h); + fail_unless (list != NULL); + fail_unless_equals_int (gst_buffer_list_length (list), 2); + for (int i = 0; i < 2; ++i) { + buffer = gst_buffer_list_get(list, i); + fail_unless_equals_int(gst_buffer_get_size(buffer), i*17) + } + gst_buffer_list_unref (list); + + /* Push list, pull individual buffers */ + push_list = gst_buffer_list_new_sized (3); + for (int i = 0; i < 3; ++i) { + buffer = gst_buffer_new_allocate (NULL, i * 127, NULL); + gst_buffer_list_add (push_list, buffer); + } + + gst_harness_push_list (h, push_list); + + for (int i = 0; i < 3; ++i) { + buffer = gst_harness_pull (h); + fail_unless_equals_int (gst_buffer_get_size (buffer), i * 127); + fail_unless (buffer != NULL); + gst_buffer_unref (buffer); + } + + /* Push a buffer */ + buffer = gst_buffer_new_allocate (NULL, 384, NULL); + gst_harness_push (h, buffer); + + /* Push a list */ + push_list = gst_buffer_list_new_sized (4); + for (int i = 0; i < 4; ++i) { + buffer = gst_buffer_new_allocate (NULL, i * 937, NULL); + gst_buffer_list_add (push_list, buffer); + } + gst_harness_push_list (h, push_list); + + /* Try to pull list, which should fail as a buffer is at the front of the queue. */ + list = gst_harness_pull_list (h); + fail_unless (list == NULL); + + /* Pull the buffer at the front of the queue */ + buffer = gst_harness_pull (h); + fail_unless_equals_int (gst_buffer_get_size (buffer), 384); + fail_unless (buffer != NULL); + gst_buffer_unref (buffer); + + /* Pull the list that is now at the front of the queue */ + list = gst_harness_pull_list (h); + fail_unless (list != NULL); + fail_unless_equals_int (gst_buffer_list_length (list), 4); + for (int i = 0; i < 2; ++i) { + buffer = gst_buffer_list_get(list, i); + fail_unless_equals_int(gst_buffer_get_size(buffer), i*937); + } + gst_buffer_list_unref (list); + + gst_harness_teardown (h); +} + +GST_END_TEST; + + static Suite * gst_harness_suite (void) { @@ -304,6 +403,7 @@ gst_harness_suite (void) tcase_add_test (tc_chain, test_harness_empty); tcase_add_test (tc_chain, test_harness_element_ref); tcase_add_test (tc_chain, test_src_harness); + tcase_add_test (tc_chain, test_src_harness_buflist); tcase_add_test (tc_chain, test_src_harness_no_forwarding); tcase_add_test (tc_chain, test_add_sink_harness_without_sinkpad); diff --git a/subprojects/gstreamer/tests/check/libs/gsttxfeedback.c b/subprojects/gstreamer/tests/check/libs/gsttxfeedback.c new file mode 100644 index 00000000000..39f184c8ddf --- /dev/null +++ b/subprojects/gstreamer/tests/check/libs/gsttxfeedback.c @@ -0,0 +1,116 @@ +/* GStreamer + * Copyright (C) <2020> Havard Graff + * + * gsttxfeedback.c: Unit test for the TX Feedback + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +typedef struct +{ + GObject object; + guint64 buffer_id; + GstClockTime ts; +} GDummyObject; + +typedef GObjectClass GDummyObjectClass; + +static void +g_dummy_object_tx_feedback (GstTxFeedback * parent, guint64 buffer_id, + GstClockTime ts) +{ + GDummyObject *obj = (GDummyObject *) parent; + obj->buffer_id = buffer_id; + obj->ts = ts; +} + +static void +g_dummy_object_tx_feedback_init (gpointer g_iface, + G_GNUC_UNUSED gpointer iface_data) +{ + GstTxFeedbackInterface *iface = g_iface; + iface->tx_feedback = g_dummy_object_tx_feedback; +} + +GType g_dummy_object_get_type (void); +G_DEFINE_TYPE_WITH_CODE (GDummyObject, g_dummy_object, + G_TYPE_OBJECT, G_IMPLEMENT_INTERFACE (GST_TYPE_TX_FEEDBACK, + g_dummy_object_tx_feedback_init)); + +#define G_TYPE_DUMMY_OBJECT g_dummy_object_get_type() + +static void +g_dummy_object_class_init (G_GNUC_UNUSED GDummyObjectClass * klass) +{ +} + +static void +g_dummy_object_init (G_GNUC_UNUSED GDummyObject * enc) +{ +} + +GST_START_TEST (test_basic) +{ + GstTxFeedbackMeta *meta; + GDummyObject *obj; + GstBuffer *buf; + guint64 buffer_id = 42; + GstClockTime ts = 123456789; + + obj = g_object_new (G_TYPE_DUMMY_OBJECT, NULL); + fail_unless (obj); + + buf = gst_buffer_new (); + + /* add TxFeedback buffer meta, with a unique ID and the receving object */ + gst_buffer_add_tx_feedback_meta (buf, buffer_id, GST_TX_FEEDBACK (obj)); + + /* verify the meta is on the buffer */ + meta = gst_buffer_get_tx_feedback_meta (buf); + fail_unless (meta); + + /* now set the transmit time of this buffer */ + gst_tx_feedback_meta_set_tx_time (meta, ts); + + /* and verify that the object was notified of this buffer being sent */ + fail_unless_equals_int (obj->buffer_id, buffer_id); + fail_unless_equals_int (obj->ts, ts); + + gst_buffer_unref (buf); + g_object_unref (obj); +} + +GST_END_TEST; + +static Suite * +gst_tx_feedback_suite (void) +{ + Suite *s = suite_create ("GstTxFeedback"); + TCase *tc_chain = tcase_create ("generic tests"); + + suite_add_tcase (s, tc_chain); + tcase_add_test (tc_chain, test_basic); + + return s; +} + +GST_CHECK_MAIN (gst_tx_feedback); diff --git a/subprojects/gstreamer/tests/check/libs/test_transform.c b/subprojects/gstreamer/tests/check/libs/test_transform.c index 534599b6c18..d380f9d4aba 100644 --- a/subprojects/gstreamer/tests/check/libs/test_transform.c +++ b/subprojects/gstreamer/tests/check/libs/test_transform.c @@ -165,30 +165,36 @@ result_buffer_alloc (GstPad * pad, guint64 offset, guint size, GstCaps * caps, } #endif -static TestTransData * -gst_test_trans_new (void) +static GstElement * +gst_test_trans_element_new (void) { - TestTransData *res; - GstPad *tmp; - GstPadTemplate *templ; GType type; /* we register a new sub-class for every test-run, so the class init * function is called for every test run and can be set up properly * even with CK_FORK=no */ - { - static gint counter = 0; - gchar name[100]; + static gint counter = 0; + gchar name[100]; - g_snprintf (name, sizeof (name), "GstTestTrans%d", ++counter); + g_snprintf (name, sizeof (name), "GstTestTrans%d", ++counter); - type = g_type_register_static_simple (GST_TYPE_BASE_TRANSFORM, name, - sizeof (GstTestTransClass), (GClassInitFunc) gst_test_trans_class_init, - sizeof (GstTestTrans), (GInstanceInitFunc) gst_test_trans_init, 0); - } + type = g_type_register_static_simple (GST_TYPE_BASE_TRANSFORM, name, + sizeof (GstTestTransClass), (GClassInitFunc) gst_test_trans_class_init, + sizeof (GstTestTrans), (GInstanceInitFunc) gst_test_trans_init, 0); + + return g_object_new (type, NULL); +} + + +static TestTransData * +gst_test_trans_new (void) +{ + TestTransData *res; + GstPad *tmp; + GstPadTemplate *templ; res = g_new0 (TestTransData, 1); - res->trans = g_object_new (type, NULL); + res->trans = gst_test_trans_element_new (); templ = gst_static_pad_template_get (sink_template); templ->direction = GST_PAD_SRC; diff --git a/subprojects/gstreamer/tests/check/libs/transform1.c b/subprojects/gstreamer/tests/check/libs/transform1.c index 2b9cb1e17eb..2dcd6f0f705 100644 --- a/subprojects/gstreamer/tests/check/libs/transform1.c +++ b/subprojects/gstreamer/tests/check/libs/transform1.c @@ -25,6 +25,7 @@ #endif #include #include +#include #include #include "test_transform.c" @@ -941,6 +942,170 @@ GST_START_TEST (basetransform_invalid_fixatecaps_impl) GST_END_TEST; +static GstFlowReturn +transform_ip_assert_writable (GstBaseTransform * trans, GstBuffer * buf) +{ + GST_DEBUG_OBJECT (trans, "transform called"); + + if (!gst_base_transform_is_passthrough (trans)) + fail_unless (gst_buffer_is_writable (buf)); + + return GST_FLOW_OK; +} + +static void +toggle_passthrough (gpointer data, gpointer user_data) +{ + GstBaseTransform *basetrans = GST_BASE_TRANSFORM (user_data); + + gst_base_transform_set_passthrough (basetrans, TRUE); + g_thread_yield (); + gst_base_transform_set_passthrough (basetrans, FALSE); +} + +GST_START_TEST (basetransform_stress_pt_ip) +{ + GstHarness *h[10]; + GstHarnessThread *push[10], *pt[10]; + GstElement *trans; + GstBuffer *buffer; + GstSegment segment; + GstCaps *caps; + guint i; + + klass_transform_ip = transform_ip_assert_writable; + gst_segment_init (&segment, GST_FORMAT_TIME); + caps = gst_caps_from_string ("foo/x-bar"); + buffer = gst_buffer_new (); + + for (i = 0; i < G_N_ELEMENTS (h); i++) { + trans = gst_test_trans_element_new (); + h[i] = gst_harness_new_with_element (trans, "sink", "src"); + push[i] = gst_harness_stress_push_buffer_start (h[i], caps, &segment, buffer); + pt[i] = gst_harness_stress_custom_start (h[i], NULL, toggle_passthrough, trans, 0); + g_object_unref (trans); + } + gst_caps_unref (caps); + gst_buffer_unref (buffer); + + g_usleep (G_USEC_PER_SEC/10); + + for (i = 0; i < G_N_ELEMENTS(h); i++) { + gst_harness_stress_thread_stop (pt[i]); + gst_harness_stress_thread_stop (push[i]); + gst_harness_teardown (h[i]); + } +} + +GST_END_TEST; + +static gboolean +test_harness_sink_query (GstPad * pad, GstObject * parent, GstQuery * query) +{ + GstPadQueryFunction query_func = g_object_get_qdata (G_OBJECT (pad), + g_quark_from_static_string ("test-data-harness-query-func")); + + if (GST_QUERY_TYPE (query) == GST_QUERY_ALLOCATION) { + GstPromise **p = g_object_get_qdata (G_OBJECT (pad), + g_quark_from_static_string ("test-data-promises")); + + GstPromise *padblocked = p[0]; + GstPromise *padunblocked = p[1]; + // Notifay main thread that basetransform sent allocation query + // and waite while the main thread changes the 'passthrough' flag + gst_promise_reply (padblocked, NULL); + gst_promise_wait (padunblocked); + } + return query_func (pad, parent, query); +} + +static GstFlowReturn +transform_pt_ct_alloc_query (GstBaseTransform * trans, + GstBuffer * in, GstBuffer * out) +{ + return GST_FLOW_OK; +} + +static gboolean +reconfigure_each_pad (GstElement * element, + GstPad * pad, + gpointer user_data) +{ + (void) user_data; + gst_pad_mark_reconfigure (pad); + return TRUE; +} + +static GstBuffer * +before_buffer_push (GstHarness * h, gpointer data) +{ + gst_element_foreach_src_pad(h->element, reconfigure_each_pad, NULL); + gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (h->element), FALSE); + return gst_buffer_ref (GST_BUFFER_CAST (data)); +} + +/* Test for capability to modification/read basetransform privat data + * while it is wating for query results (in this case allocation query). +*/ +GST_START_TEST (basetransform_stress_pt_ct_alloc_query) +{ + GstHarness *h = NULL; + GstHarnessThread *push = NULL; + GstElement *trans = NULL; + GstBuffer *buffer = NULL; + GstSegment segment; + GstCaps *caps = NULL; + GstPromise *padblocked = gst_promise_new (); + GstPromise *padunblocked = gst_promise_new (); + GstPromise *p[2] = {padblocked, padunblocked}; + GstPadQueryFunction old_query_func = NULL; + + klass_transform = transform_pt_ct_alloc_query; + gst_segment_init (&segment, GST_FORMAT_TIME); + caps = gst_caps_from_string ("foo/x-bar"); + buffer = gst_buffer_new (); + + trans = gst_test_trans_element_new (); + h = gst_harness_new_with_element (trans, "sink", "src"); + + old_query_func = GST_PAD_QUERYFUNC (h->sinkpad); + // Store the old harness query handler + g_object_set_qdata (G_OBJECT (h->sinkpad), + g_quark_from_static_string ("test-data-harness-query-func"), + old_query_func); + + g_object_set_qdata (G_OBJECT (h->sinkpad), + g_quark_from_static_string ("test-data-promises"), p); + + // Set the new harness query handler to intercect the allocation + // queries. + gst_pad_set_query_function (h->sinkpad, test_harness_sink_query); + + // Start the dedicated push buffer thread. + push = gst_harness_stress_push_buffer_with_cb_start(h, caps, &segment, + before_buffer_push, buffer, (GDestroyNotify) gst_buffer_unref); + + // Wait for notification from harness pad when basetransform sents allocation query. + gst_promise_wait (padblocked); + // Now we should be able to modifay the basetransform passthrough mode. + gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (h->element), TRUE); + // Notifay harness thread to continue buffer processing + gst_promise_reply (padunblocked, NULL); + + gst_harness_stress_thread_stop (push); + + GST_PAD_QUERYFUNC (h->sinkpad) = old_query_func; + + gst_buffer_unref (buffer); + gst_caps_unref (caps); + g_object_unref (trans); + gst_harness_teardown (h); + gst_promise_unref (padblocked); + gst_promise_unref (padunblocked); +} + +GST_END_TEST; + static void transform1_setup (void) { @@ -981,6 +1146,9 @@ gst_basetransform_suite (void) tcase_add_test (tc, basetransform_chain_ct1); tcase_add_test (tc, basetransform_chain_ct2); tcase_add_test (tc, basetransform_chain_ct3); + /* stress */ + tcase_add_test (tc, basetransform_stress_pt_ip); + tcase_add_test (tc, basetransform_stress_pt_ct_alloc_query); tcase_add_test (tc, basetransform_invalid_fixatecaps_impl); diff --git a/subprojects/gstreamer/tests/check/meson.build b/subprojects/gstreamer/tests/check/meson.build index 65402791aac..8c94caf2796 100644 --- a/subprojects/gstreamer/tests/check/meson.build +++ b/subprojects/gstreamer/tests/check/meson.build @@ -57,6 +57,7 @@ core_tests = [ [ 'libs/baseparse.c' ], [ 'libs/basesrc.c', not gst_registry ], [ 'libs/basesink.c', not gst_registry ], + [ 'libs/baseidlesrc.c', not gst_registry ], [ 'libs/bitreader.c' ], [ 'libs/bitwriter.c' ], [ 'libs/bytereader.c' ], @@ -71,6 +72,7 @@ core_tests = [ [ 'libs/gstnetclientclock.c' ], [ 'libs/gstnettimeprovider.c' ], [ 'libs/gsttestclock.c' ], + [ 'libs/gsttxfeedback.c' ], [ 'libs/libsabi.c' ], [ 'libs/sparsefile.c' ], [ 'libs/transform1.c' ], diff --git a/tests/check/elements/rtmp.c b/tests/check/elements/rtmp.c new file mode 100644 index 00000000000..04315b5c41e --- /dev/null +++ b/tests/check/elements/rtmp.c @@ -0,0 +1,141 @@ +/* GStreamer + * + * unit test for rtmp + * + * Copyright (C) <2016> Havard Graff + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include +#include + +#include +#include +#include + +static gint +add_listen_fd (int port) +{ + gint fd = socket (AF_INET6, SOCK_STREAM, 0); + g_assert_cmpint (fd, >=, 0); + + int sock_optval = 1; + setsockopt (fd, SOL_SOCKET, SO_REUSEADDR, &sock_optval, sizeof (sock_optval)); + + struct sockaddr_in6 sin; + memset (&sin, 0, sizeof (struct sockaddr_in6)); + sin.sin6_family = AF_INET6; + sin.sin6_port = htons (port); + sin.sin6_addr = in6addr_any; + + g_assert (bind (fd, (struct sockaddr *) &sin, sizeof (sin)) >= 0); + listen (fd, 10); + + return fd; +} + + +GST_START_TEST (rtmpsink_unlock) +{ + gint fd; + GstHarness *h; + GTimer *timer; + + fd = add_listen_fd (22000); + h = gst_harness_new_parse + ("queue ! rtmpsink location=rtmp://localhost:22000/app/streamname1"); + gst_harness_set_src_caps_str (h, "video/x-flv"); + + timer = g_timer_new (); + gst_harness_push (h, gst_buffer_new ()); + + if (__i__ == 1) + g_usleep (G_USEC_PER_SEC / 10); + + gst_harness_teardown (h); + fail_unless (g_timer_elapsed (timer, NULL) < 2.0); + + close (fd); +} + +GST_END_TEST; + +GST_START_TEST (rtmpsink_unlock_race) +{ + GstHarness *h = gst_harness_new_parse ("rtmpsink location=rtmp://a/b/c"); + GstHarnessThread *statechange, *push; + GstSegment segment; + GstCaps *caps = gst_caps_from_string ("video/x-flv"); + GstBuffer *buf = gst_buffer_new (); + + gst_segment_init (&segment, GST_FORMAT_TIME); + + statechange = gst_harness_stress_statechange_start_full (h, 1); + push = gst_harness_stress_push_buffer_start (h, caps, &segment, buf); + + g_usleep (G_USEC_PER_SEC * 1); + + gst_harness_stress_thread_stop (statechange); + gst_harness_stress_thread_stop (push); + + gst_caps_unref (caps); + gst_buffer_unref (buf); + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST(rtmpsrc_unlock) +{ + gint fd; + GstHarness *h; + GTimer *timer; + + fd = add_listen_fd (23000); + h = gst_harness_new_parse ( + "rtmpsrc location=rtmp://localhost:23000/app/streamname1"); + + timer = g_timer_new (); + gst_harness_play (h); + + if (__i__ == 1) + g_usleep (G_USEC_PER_SEC / 10); + + gst_harness_teardown (h); + fail_unless (g_timer_elapsed (timer, NULL) < 2.0); + + g_timer_destroy (timer); + close (fd); +} +GST_END_TEST; + +static Suite * +rtmp_suite (void) +{ + Suite *s = suite_create ("rtmp"); + TCase *tc_chain = tcase_create ("general"); + suite_add_tcase (s, tc_chain); + + tcase_add_loop_test (tc_chain, rtmpsink_unlock, 0, 2); + tcase_add_test (tc_chain, rtmpsink_unlock_race); + + tcase_add_loop_test (tc_chain, rtmpsrc_unlock, 0, 2); + + return s; +} + +GST_CHECK_MAIN (rtmp); diff --git a/tests/check/elements/speex.c b/tests/check/elements/speex.c new file mode 100644 index 00000000000..7db3cc2c9d7 --- /dev/null +++ b/tests/check/elements/speex.c @@ -0,0 +1,261 @@ +/* GStreamer + * + * unit test for speex + * + * Copyright (C) <2017> Havard Graff + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +GST_START_TEST (test_encoder_timestamp) +{ + GstHarness * h = gst_harness_new ("speexenc"); + GstBuffer * buf; + gst_harness_add_src_parse (h, "audiotestsrc is-live=1 samplesperbuffer=320 ! " + "capsfilter caps=\"audio/x-raw,format=S16LE,rate=16000\"", TRUE); + + /* push 20ms of audio-data */ + gst_harness_push_from_src (h); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (0, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (0, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (20 * GST_MSECOND, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + /* push another 20ms of audio-data */ + gst_harness_push_from_src (h); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (20 * GST_MSECOND, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (20 * GST_MSECOND, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + gst_harness_teardown (h); +} +GST_END_TEST; + +GST_START_TEST (test_encoder_to_decoder_timestamp) +{ + GstHarness * h = gst_harness_new_parse ("speexenc ! speexdec"); + GstBuffer * buf; + gst_harness_add_src_parse (h, "audiotestsrc is-live=1 samplesperbuffer=320 ! " + "capsfilter caps=\"audio/x-raw,format=S16LE,rate=16000\"", TRUE); + + /* push 20ms of audio-data */ + gst_harness_push_from_src (h); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (20 * GST_MSECOND, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + /* push another 20ms of audio-data */ + gst_harness_push_from_src (h); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (20 * GST_MSECOND, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (20 * GST_MSECOND, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + gst_harness_teardown (h); +} +GST_END_TEST; + +GST_START_TEST (test_headers_in_caps) +{ + GstHarness * h = gst_harness_new ("speexdec"); + gst_harness_add_src_parse (h, "audiotestsrc is-live=1 samplesperbuffer=320 ! " + "capsfilter caps=\"audio/x-raw,format=S16LE,rate=16000\" ! " + "speexenc", TRUE); + + /* the first audio-buffer produces 3 packets: + streamheader, vorbiscomments and the encoded buffer */ + gst_harness_src_crank_and_push_many (h, 1, 3); + + /* verify the decoder produces exactly one decoded buffer */ + gst_buffer_unref (gst_harness_pull (h)); + fail_unless_equals_int (0, gst_harness_buffers_in_queue (h)); + + gst_harness_teardown (h); +} +GST_END_TEST; + +GST_START_TEST (test_headers_in_buffers) +{ + GstHarness * h = gst_harness_new ("speexdec"); + gst_harness_set_src_caps_str (h, "audio/x-speex"); + + /* turn off forwarding to avoid getting caps from the src-harness, + and hence not accessing the streamheaders through the caps */ + gst_harness_set_forwarding (h, FALSE); + + gst_harness_add_src_parse (h, "audiotestsrc is-live=1 samplesperbuffer=320 ! " + "capsfilter caps=\"audio/x-raw,format=S16LE,rate=16000\" ! " + "speexenc", TRUE); + + /* the first audio-buffer produces 3 packets: + streamheader, vorbiscomments and the encoded buffer */ + gst_harness_src_crank_and_push_many (h, 1, 3); + + /* verify the decoder produces exactly one decoded buffer */ + gst_buffer_unref (gst_harness_pull (h)); + fail_unless_equals_int (0, gst_harness_buffers_in_queue (h)); + + gst_harness_teardown (h); +} +GST_END_TEST; + +GST_START_TEST (test_headers_not_in_buffers) +{ + GstHarness * h = gst_harness_new ("speexdec"); + + gst_harness_add_src_parse (h, "audiotestsrc is-live=1 samplesperbuffer=320 ! " + "capsfilter caps=\"audio/x-raw,format=S16LE,rate=16000\" ! " + "speexenc", TRUE); + + /* the first audio-buffer produces 3 packets: + streamheader, vorbiscomments and the encoded buffer */ + gst_harness_src_crank_and_push_many (h, 1, 0); + + /* now remove the streamheader and vorbiscomment */ + gst_buffer_unref (gst_harness_pull (h->src_harness)); + gst_buffer_unref (gst_harness_pull (h->src_harness)); + + /* and just push the encoded buffer to the decoder */ + gst_harness_src_crank_and_push_many (h, 0, 1); + + /* verify the decoder produces exactly one decoded buffer */ + gst_buffer_unref (gst_harness_pull (h)); + fail_unless_equals_int (0, gst_harness_buffers_in_queue (h)); + + gst_harness_teardown (h); +} +GST_END_TEST; + +GST_START_TEST (test_headers_from_flv) +{ + GstHarness * h = gst_harness_new ("speexdec"); + + /* set caps with the same streamheader flvdemux would add */ + gst_harness_set_src_caps_str (h, "audio/x-speex, " + "streamheader=(buffer)< 5370656578202020312e312e3132000000000000000000000" + "00000000100000050000000803e0000010000000400000001000000ffffffff500000000" + "000000001000000000000000000000000000000, " + "0b0000004e6f20636f6d6d656e74730000000001 >, " + "rate=(int)16000, channels=(int)1"); + + /* turn off forwarding to avoid getting caps from the src-harness, + and force it to use the flv-streamheaders we have set in the caps */ + gst_harness_set_forwarding (h, FALSE); + + gst_harness_add_src_parse (h, "audiotestsrc is-live=1 samplesperbuffer=320 ! " + "capsfilter caps=\"audio/x-raw,format=S16LE,rate=16000\" ! " + "speexenc", TRUE); + + /* the first audio-buffer produces 3 packets: + streamheader, vorbiscomments and the encoded buffer */ + gst_harness_src_crank_and_push_many (h, 1, 3); + + /* verify the decoder produces exactly one decoded buffer */ + gst_buffer_unref (gst_harness_pull (h)); + fail_unless_equals_int (0, gst_harness_buffers_in_queue (h)); + + gst_harness_teardown (h); +} +GST_END_TEST + +GST_START_TEST (test_new_segment_before_first_output_buffer) +{ + GstHarness *h = gst_harness_new ("speexenc"); + GstBuffer *buf; + GstSegment segment; + + /* 10 ms buffers */ + gst_harness_add_src_parse (h, "audiotestsrc is-live=1 samplesperbuffer=160 ! " + "capsfilter caps=\"audio/x-raw,format=S16LE,rate=16000\"", TRUE); + + /* push 10 ms of audio-data */ + gst_harness_push_from_src (h); + + /* send a second segment event */ + gst_segment_init (&segment, GST_FORMAT_TIME); + fail_unless (gst_harness_push_event (h, gst_event_new_segment (&segment))); + + /* push 10 ms of audio-data */ + gst_harness_push_from_src (h); + + /* pull 20 ms of encoded audio-data with headers */ + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (0, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (0, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + buf = gst_harness_pull (h); + fail_unless_equals_uint64 (0, GST_BUFFER_PTS (buf)); + fail_unless_equals_uint64 (20 * GST_MSECOND, GST_BUFFER_DURATION (buf)); + gst_buffer_unref (buf); + + gst_harness_teardown (h); +} +GST_END_TEST; + + +static Suite * +speex_suite (void) +{ + Suite *s = suite_create ("speex"); + TCase *tc_chain; + + suite_add_tcase (s, (tc_chain = tcase_create ("timestamping"))); + tcase_add_test (tc_chain, test_encoder_timestamp); + tcase_add_test (tc_chain, test_encoder_to_decoder_timestamp); + + suite_add_tcase (s, (tc_chain = tcase_create ("headers"))); + tcase_add_test (tc_chain, test_headers_in_caps); + tcase_add_test (tc_chain, test_headers_in_buffers); + tcase_add_test (tc_chain, test_headers_not_in_buffers); + tcase_add_test (tc_chain, test_headers_from_flv); + + suite_add_tcase (s, (tc_chain = tcase_create ("events"))); + tcase_add_test (tc_chain, test_new_segment_before_first_output_buffer); + + + return s; +} + +GST_CHECK_MAIN (speex) diff --git a/tests/check/libs/gstpriqueue.c b/tests/check/libs/gstpriqueue.c new file mode 100644 index 00000000000..30d40c6a7e6 --- /dev/null +++ b/tests/check/libs/gstpriqueue.c @@ -0,0 +1,420 @@ +/* GStreamer + * Copyright (C) 2016 Pexip AS + * Erlend Graff + * + * gstpriqueue.c: unit tests for GstPriQueue + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include +#include +#include + +struct item_data +{ + gint key; + gint inserted; + GstPriQueueElem pq_elem; +}; + +#define ELEM_TO_ITEM(elem) (GST_CONTAINER_OF (elem, struct item_data, pq_elem)) + +static gint +compare_items (const GstPriQueueElem * elem_a, const GstPriQueueElem * elem_b, + G_GNUC_UNUSED gpointer user_data) +{ + return ELEM_TO_ITEM (elem_a)->key - ELEM_TO_ITEM (elem_b)->key; +} + +static gint +compare_items_with_check (const GstPriQueueElem * elem_a, + const GstPriQueueElem * elem_b, gpointer user_data) +{ + fail_unless_equals_int (-12345, GPOINTER_TO_INT (user_data)); + return compare_items (elem_a, elem_b, NULL); +} + +GST_START_TEST (test_random_modifications) +{ + const gint num_items = 1023; + const gint num_iter = 500; + const gint key_min = -1000; + const gint key_max = 1000; + struct item_data *items; + GstPriQueueElem *lookup_elem; + GstPriQueue *pq; + GRand *rand; + gint i, minidx, idx, num_inserted; + + rand = g_rand_new_with_seed (0); + items = g_new0 (struct item_data, num_items); + + pq = gst_pri_queue_create (compare_items_with_check, + GINT_TO_POINTER (-12345)); + + num_inserted = 0; + for (i = 0; i < num_iter; i++) { + /* Ensure valid */ + fail_unless (gst_pri_queue_is_valid (pq)); + + /* Test get_min and size */ + + fail_unless_equals_int (num_inserted, gst_pri_queue_size (pq)); + + minidx = -1; + lookup_elem = gst_pri_queue_get_min (pq); + fail_unless_equals_int (! !num_inserted, ! !lookup_elem); + + if (num_inserted) { + minidx = ELEM_TO_ITEM (lookup_elem) - items; + for (idx = 0; idx < num_items; idx++) { + /* There may be multiple "smallest" elements (i.e. other elements may + * have the same priority/key as the one we just retrieved), hence the + * use of `>=`. + */ + if (items[idx].inserted) + fail_unless (items[idx].key >= items[minidx].key); + } + } + + /* Test insert, remove, and update */ + + idx = g_rand_int_range (rand, 0, num_items); + + if (!items[idx].inserted) { + /* Just to prevent insertions from dominating the types of operations + * performed. + */ + if (g_rand_int_range (rand, 0, 5) != 0) { + i--; + continue; + } + + items[idx].key = g_rand_int_range (rand, key_min, key_max); + gst_pri_queue_insert (pq, &items[idx].pq_elem); + items[idx].inserted = TRUE; + num_inserted++; + } else { + switch (g_rand_int_range (rand, 0, 4)) { + case 0: + gst_pri_queue_remove (pq, &items[idx].pq_elem); + items[idx].inserted = FALSE; + num_inserted--; + break; + case 1: + lookup_elem = gst_pri_queue_pop_min (pq); + fail_unless_equals_pointer (&items[minidx].pq_elem, lookup_elem); + items[minidx].inserted = FALSE; + num_inserted--; + break; + case 2: + gst_pri_queue_remove (pq, &items[idx].pq_elem); + items[idx].key = g_rand_int_range (rand, key_min, key_max); + gst_pri_queue_insert (pq, &items[idx].pq_elem); + break; + case 3: + items[idx].key = g_rand_int_range (rand, key_min, key_max); + gst_pri_queue_update (pq, &items[idx].pq_elem); + break; + default: + g_assert_not_reached (); + break; + } + } + } + + while (TRUE) { + /* Ensure valid */ + fail_unless (gst_pri_queue_is_valid (pq)); + + /* Test get_min and size */ + + fail_unless_equals_int (num_inserted, gst_pri_queue_size (pq)); + + lookup_elem = gst_pri_queue_get_min (pq); + fail_unless_equals_int (! !num_inserted, ! !lookup_elem); + + if (num_inserted) { + minidx = ELEM_TO_ITEM (lookup_elem) - items; + for (idx = 0; idx < num_items; idx++) { + /* There may be multiple "smallest" elements (i.e. other elements may + * have the same priority/key as the one we just retrieved), hence the + * use of `>=`. + */ + if (items[idx].inserted) + fail_unless (items[idx].key >= items[minidx].key); + } + } + + if (!num_inserted) + break; + + /* Test remove and update */ + + /* Find random inserted item */ + idx = g_rand_int_range (rand, 0, num_items); + while (!items[idx].inserted) + idx = (idx + 1) % num_items; + + switch (g_rand_int_range (rand, 0, 2)) { + case 0: + gst_pri_queue_remove (pq, &items[idx].pq_elem); + items[idx].inserted = FALSE; + num_inserted--; + break; + case 1: + items[idx].key = g_rand_int_range (rand, key_min, key_max); + gst_pri_queue_update (pq, &items[idx].pq_elem); + break; + default: + g_assert_not_reached (); + break; + } + } + + gst_pri_queue_destroy (pq, NULL); + + g_free (items); + g_rand_free (rand); +} + +GST_END_TEST; + +#define NUM_ITEMS 127 + +static gint +compare_pointers (const GstPriQueueElem * elem_a, + const GstPriQueueElem * elem_b, G_GNUC_UNUSED gpointer user_data) +{ + return elem_a - elem_b; +} + +GST_START_TEST (test_sorted_insertion) +{ + GstPriQueueElem *elems; + GstPriQueue *pq; + gint i; + + elems = g_new (GstPriQueueElem, NUM_ITEMS); + + pq = gst_pri_queue_create (compare_pointers, NULL); + for (i = 0; i < NUM_ITEMS; i++) { + gst_pri_queue_insert (pq, &elems[i]); + fail_unless_equals_int (i + 1, gst_pri_queue_size (pq)); + fail_unless_equals_pointer (&elems[0], gst_pri_queue_get_min (pq)); + fail_unless (gst_pri_queue_is_valid (pq)); + } + gst_pri_queue_destroy (pq, NULL); + + pq = gst_pri_queue_create (compare_pointers, NULL); + for (i = NUM_ITEMS - 1; i >= 0; i--) { + gst_pri_queue_insert (pq, &elems[i]); + fail_unless_equals_int (NUM_ITEMS - i, gst_pri_queue_size (pq)); + fail_unless_equals_pointer (&elems[i], gst_pri_queue_get_min (pq)); + fail_unless (gst_pri_queue_is_valid (pq)); + } + gst_pri_queue_destroy (pq, NULL); + + g_free (elems); +} + +GST_END_TEST; + +GST_START_TEST (test_sorted_removal) +{ + GstPriQueueElem *elems; + GstPriQueue *pq; + gint i; + + elems = g_new (GstPriQueueElem, NUM_ITEMS); + pq = gst_pri_queue_create (compare_pointers, NULL); + + for (i = 0; i < NUM_ITEMS; i++) + gst_pri_queue_insert (pq, &elems[i]); + + for (i = 0; i < NUM_ITEMS; i++) { + fail_unless_equals_pointer (&elems[i], gst_pri_queue_get_min (pq)); + gst_pri_queue_remove (pq, &elems[i]); + fail_unless_equals_int (NUM_ITEMS - 1 - i, gst_pri_queue_size (pq)); + fail_unless (gst_pri_queue_is_valid (pq)); + } + + fail_unless_equals_pointer (NULL, gst_pri_queue_get_min (pq)); + + for (i = 0; i < NUM_ITEMS; i++) + gst_pri_queue_insert (pq, &elems[i]); + + for (i = NUM_ITEMS - 1; i >= 0; i--) { + fail_unless_equals_pointer (&elems[0], gst_pri_queue_get_min (pq)); + gst_pri_queue_remove (pq, &elems[i]); + fail_unless_equals_int (i, gst_pri_queue_size (pq)); + fail_unless (gst_pri_queue_is_valid (pq)); + } + + fail_unless_equals_pointer (NULL, gst_pri_queue_get_min (pq)); + + gst_pri_queue_destroy (pq, NULL); + g_free (elems); +} + +GST_END_TEST; + +GST_START_TEST (test_pop_min) +{ + GstPriQueueElem *elems; + GstPriQueue *pq; + gint i; + + elems = g_new (GstPriQueueElem, NUM_ITEMS); + pq = gst_pri_queue_create (compare_pointers, NULL); + + for (i = 0; i < NUM_ITEMS; i++) + gst_pri_queue_insert (pq, &elems[i]); + + for (i = 0; i < NUM_ITEMS; i++) { + fail_unless_equals_pointer (&elems[i], gst_pri_queue_pop_min (pq)); + fail_unless_equals_int (NUM_ITEMS - 1 - i, gst_pri_queue_size (pq)); + fail_unless (gst_pri_queue_is_valid (pq)); + } + + fail_unless_equals_pointer (NULL, gst_pri_queue_pop_min (pq)); + + gst_pri_queue_destroy (pq, NULL); + g_free (elems); +} + +GST_END_TEST; + +static void +increase_inserted (gpointer elem) +{ + struct item_data *item = ELEM_TO_ITEM (elem); + item->inserted += 1; +} + +GST_START_TEST (test_destroy) +{ + struct item_data *items; + GstPriQueue *pq; + gint i; + + items = g_new (struct item_data, NUM_ITEMS); + + pq = gst_pri_queue_create (compare_items, NULL); + for (i = 0; i < NUM_ITEMS; i++) { + /* Note: we abuse the `inserted` field to keep track of how many times each + * the destroy func is called for each element. + */ + items[i].inserted = 0; + items[i].key = i; + gst_pri_queue_insert (pq, &items[i].pq_elem); + } + + gst_pri_queue_destroy (pq, increase_inserted); + + for (i = 0; i < NUM_ITEMS; i++) { + /* Each element should be visited by destroy func exactly once */ + fail_unless_equals_int (1, items[i].inserted); + } + + g_free (items); +} + +GST_END_TEST; + +GST_START_TEST (test_meld) +{ + GstPriQueueElem *elems; + GstPriQueue *pqa, *pqb, *pq_meld; + gint i; + + elems = g_new (GstPriQueueElem, NUM_ITEMS); + pqa = gst_pri_queue_create (compare_pointers, NULL); + pqb = gst_pri_queue_create (compare_pointers, NULL); + + for (i = 0; i < NUM_ITEMS / 2; i++) + gst_pri_queue_insert (pqa, &elems[i]); + + for (; i < NUM_ITEMS; i++) + gst_pri_queue_insert (pqb, &elems[i]); + + pq_meld = gst_pri_queue_meld (pqa, pqb); + fail_unless_equals_pointer (pqa, pq_meld); + + fail_unless_equals_int (NUM_ITEMS, gst_pri_queue_size (pqa)); + fail_unless (gst_pri_queue_is_valid (pqa)); + + for (i = 0; i < NUM_ITEMS; i++) { + fail_unless_equals_pointer (&elems[i], gst_pri_queue_pop_min (pqa)); + fail_unless_equals_int (NUM_ITEMS - 1 - i, gst_pri_queue_size (pqa)); + fail_unless (gst_pri_queue_is_valid (pqa)); + } + + gst_pri_queue_destroy (pqa, NULL); + + g_free (elems); +} + +GST_END_TEST; + +GST_START_TEST (test_iter) +{ + struct item_data *items; + GstPriQueueElem *elem; + GstPriQueueIter iter; + GstPriQueue *pq; + gint i; + + items = g_new0 (struct item_data, NUM_ITEMS); + pq = gst_pri_queue_create (compare_items, NULL); + + for (i = 0; i < NUM_ITEMS; i++) + gst_pri_queue_insert (pq, &items[i].pq_elem); + + gst_pri_queue_iter_init (&iter, pq); + while (gst_pri_queue_iter_next (&iter, &elem)) + ELEM_TO_ITEM (elem)->inserted += 1; + + for (i = 0; i < NUM_ITEMS; i++) + fail_unless_equals_int (1, items[i].inserted); + + gst_pri_queue_destroy (pq, NULL); + g_free (items); +} + +GST_END_TEST; + +static Suite * +gst_pri_queue_suite (void) +{ + Suite *s = suite_create ("GstPriQueue"); + TCase *tc_chain = tcase_create ("general"); + + suite_add_tcase (s, tc_chain); + + tcase_add_test (tc_chain, test_random_modifications); + tcase_add_test (tc_chain, test_sorted_insertion); + tcase_add_test (tc_chain, test_sorted_removal); + tcase_add_test (tc_chain, test_pop_min); + tcase_add_test (tc_chain, test_destroy); + tcase_add_test (tc_chain, test_meld); + tcase_add_test (tc_chain, test_iter); + + return s; +} + +GST_CHECK_MAIN (gst_pri_queue);