ngx-live: fix audio ends before video issue

erankor · erankor · commit cb3103388f4d · 2024-06-08T22:16:15.000+03:00
the segmenter may try to cut a segment at the audio end timestamp, there
is no video key frame at this timestamp - the video segments come out
empty, and some of the audio frames are discarded. then it tries to
create another segment, but the segments of all tracks come out empty,
and the channel is failed.
the fix is to avoid averaging the min/max split pts when the span is too
large. in this case, the target pts will be the audio end pts, the video
segments will come out empty, and the trailing audio will be discarded.
diff --git a/nginx-live-module/src/ngx_live_segmenter.c b/nginx-live-module/src/ngx_live_segmenter.c
@@ -59,6 +59,7 @@ typedef struct {
 
     ngx_msec_t                        candidate_margin;
     ngx_msec_t                        keyframe_alignment_margin;
+    ngx_msec_t                        max_span_average;
 
     ngx_uint_t                        ready_threshold;
     ngx_uint_t                        initial_ready_threshold;
@@ -208,6 +209,7 @@ typedef struct {
 
     uint32_t                          candidate_margin;
     uint32_t                          keyframe_alignment_margin;
+    uint32_t                          max_span_average;
 
     uint32_t                          ready_duration;
     uint32_t                          initial_ready_duration;
@@ -309,6 +311,13 @@ static ngx_command_t  ngx_live_segmenter_commands[] = {
       offsetof(ngx_live_segmenter_preset_conf_t, keyframe_alignment_margin),
       NULL },
 
+    { ngx_string("segmenter_max_span_average"),
+      NGX_LIVE_MAIN_CONF|NGX_LIVE_PRESET_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_msec_slot,
+      NGX_LIVE_PRESET_CONF_OFFSET,
+      offsetof(ngx_live_segmenter_preset_conf_t, max_span_average),
+      NULL },
+
     { ngx_string("segmenter_ready_threshold"),
       NGX_LIVE_MAIN_CONF|NGX_LIVE_PRESET_CONF|NGX_CONF_TAKE1,
       ngx_conf_set_num_slot,
@@ -2391,7 +2400,13 @@ ngx_live_segmenter_get_segment_times(ngx_live_channel_t *channel,
             continue;
         }
 
-        cur_pts = (max[i] + min[i]) / 2;
+        if (max[i] - min[i] < cctx->max_span_average) {
+            cur_pts = (max[i] + min[i]) / 2;
+
+        } else {
+            cur_pts = candidates.elts[i].pts;
+        }
+
         if (target_pts == NGX_LIVE_INVALID_PTS ||
             ngx_abs_diff(cur_pts, boundary_pts) <
             ngx_abs_diff(target_pts, boundary_pts))
@@ -3629,6 +3644,8 @@ ngx_live_segmenter_channel_init(ngx_live_channel_t *channel, void *ectx)
         spcf->candidate_margin, 1000, channel->timescale);
     cctx->keyframe_alignment_margin = ngx_live_rescale_time(
         spcf->keyframe_alignment_margin, 1000, channel->timescale);
+    cctx->max_span_average = ngx_live_rescale_time(
+        spcf->max_span_average, 1000, channel->timescale);
 
     cctx->create.data = channel;
     cctx->create.handler = ngx_live_segmenter_create_handler;
@@ -3890,6 +3907,7 @@ ngx_live_segmenter_create_preset_conf(ngx_conf_t *cf)
 
     conf->candidate_margin = NGX_CONF_UNSET_MSEC;
     conf->keyframe_alignment_margin = NGX_CONF_UNSET_MSEC;
+    conf->max_span_average = NGX_CONF_UNSET_MSEC;
 
     conf->ready_threshold = NGX_CONF_UNSET_UINT;
     conf->initial_ready_threshold = NGX_CONF_UNSET_UINT;
@@ -3945,6 +3963,9 @@ ngx_live_segmenter_merge_preset_conf(ngx_conf_t *cf, void *parent, void *child)
     ngx_conf_merge_msec_value(conf->keyframe_alignment_margin,
                               prev->keyframe_alignment_margin, 500);
 
+    ngx_conf_merge_msec_value(conf->max_span_average,
+                              prev->max_span_average, 500);
+
     ngx_conf_merge_uint_value(conf->ready_threshold,
                               prev->ready_threshold, 150);
 
diff --git a/nginx-live-module/test/nginx.conf b/nginx-live-module/test/nginx.conf
@@ -167,7 +167,7 @@ http {
         location /ksmp_proxy/ {
             internal;
             proxy_pass http://127.0.0.1:8001/ksmp/;
-            subrequest_output_buffer_size 5m;
+            subrequest_output_buffer_size 20m;
         }
 
         # curl localhost:8001/sgts/<channel_id>/<bucket_id>/seg-<segment_index>-s<track_int_id>.ts
diff --git a/nginx-live-module/test/tests/audio_ends_before_video.py b/nginx-live-module/test/tests/audio_ends_before_video.py
@@ -0,0 +1,49 @@
+from test_base import *
+
+# EXPECTED:
+#   16 sec first video + audio
+#   2 sec first video, no audio
+#   10 sec second video + audio
+
+def test(channelId=CHANNEL_ID):
+    st = KmpSendTimestamps()
+
+    nl = setupChannelTimeline(channelId)
+
+    rv = KmpMediaFileReader(TEST_VIDEO_HIGH, 0)
+    ra = KmpMediaFileReader(TEST_VIDEO_HIGH, 1)
+
+    sv, sa = createVariant(nl, VARIANT_ID, [('v1', 'video'), ('a1', 'audio')])
+
+    kmpSendStreams([
+        (rv, sv),
+        (ra, sa),
+    ], st, 17.5, realtime=5)
+
+    kmpSendStreams([
+        (rv, sv),
+        (ra, KmpNullSender()),
+    ], st, 2, realtime=5)
+
+    kmpSendEndOfStream([sv, sa])
+
+    time.sleep(2)
+
+    initialFrameId = 1000000
+
+    rv = KmpMediaFileReader(TEST_VIDEO1, 0)
+    ra = KmpMediaFileReader(TEST_VIDEO1, 1)
+
+    sv, sa = createVariant(nl, VARIANT_ID, [('v1', 'video'), ('a1', 'audio')], initialFrameId=initialFrameId)
+
+    kmpSendStreams([
+        (rv, sv),
+        (ra, sa),
+    ], st, 10, realtime=5)
+
+    kmpSendEndOfStream([sv, sa])
+
+    # deactivate the timeline
+    nl.timeline.update(NginxLiveTimeline(id=TIMELINE_ID, end_list='on'))
+
+    testDefaultStreams(channelId, __file__)
diff --git a/nginx-live-module/test/tests/ref/audio_ends_before_video-hls-fmp4.txt b/nginx-live-module/test/tests/ref/audio_ends_before_video-hls-fmp4.txt
@@ -0,0 +1,75 @@
+URL: /master.m3u8
+HEADERS: 200 application/vnd.apple.mpegurl
+BODY: #EXTM3U
+#EXT-X-INDEPENDENT-SEGMENTS
+
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=126573,AVERAGE-BANDWIDTH=114074,RESOLUTION=160x120,FRAME-RATE=15.000,CODECS="avc1.64000b,mp4a.40.2"
+index-svar1.m3u8
+
+URL: /index-svar1.m3u8
+HEADERS: 200 application/vnd.apple.mpegurl
+BODY: #EXTM3U
+#EXT-X-TARGETDURATION:8
+#EXT-X-VERSION:6
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-DISCONTINUITY-SEQUENCE:0
+#EXT-X-INDEPENDENT-SEGMENTS
+#EXT-X-ALLOW-CACHE:YES
+#EXT-X-MAP:URI="init-1-svar1.mp4"
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:00.000+00:00
+#EXTINF:8.342,
+#EXT-X-BITRATE:4971
+seg-1-svar1.m4s
+#EXTINF:7.966,
+#EXT-X-BITRATE:16215
+seg-2-svar1.m4s
+#EXT-X-DISCONTINUITY
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:16.342+00:00
+#EXTINF:3.200,
+#EXT-X-BITRATE:9141
+seg-3-svar1.m4s
+#EXT-X-DISCONTINUITY
+#EXT-X-MAP:URI="init-5-svar1.mp4"
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:19.630+00:00
+#EXTINF:3.956,
+#EXT-X-BITRATE:92
+seg-5-svar1.m4s
+#EXTINF:3.934,
+#EXT-X-BITRATE:123
+seg-6-svar1.m4s
+#EXTINF:1.989,
+#EXT-X-BITRATE:124
+seg-7-svar1.m4s
+#EXT-X-ENDLIST
+
+URL: /init-1-svar1.mp4
+HEADERS: 200 video/mp4
+BODY: SIZE: 1106, MD5: 44623fdc2bb6eb1183e22c022be3e765
+
+URL: /seg-1-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 5180236, MD5: 0d015341c4fbb773cfcc90ecbce81d0d
+
+URL: /seg-2-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 16014394, MD5: 309376ada6e59c0222bf0bab74028e1e
+
+URL: /seg-3-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 3732604, MD5: 12a71bf0970f3bafec0a7b800eb51b78
+
+URL: /init-5-svar1.mp4
+HEADERS: 200 video/mp4
+BODY: SIZE: 1106, MD5: ca1c220b1d37d697270c21bce220cdef
+
+URL: /seg-5-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 46063, MD5: 3cd142c193f879afc375070a3e30d431
+
+URL: /seg-6-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 61926, MD5: 8ab6413490c066bb6cbed25d13cdc626
+
+URL: /seg-7-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 32681, MD5: c03e8d0a6bd6bbb23a5844be89e1b6ea
diff --git a/nginx-live-module/test/tests/ref/audio_ends_before_video-hls-ts.txt b/nginx-live-module/test/tests/ref/audio_ends_before_video-hls-ts.txt
@@ -0,0 +1,65 @@
+URL: /master.m3u8
+HEADERS: 200 application/vnd.apple.mpegurl
+BODY: #EXTM3U
+#EXT-X-INDEPENDENT-SEGMENTS
+
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=144154,AVERAGE-BANDWIDTH=131383,RESOLUTION=160x120,FRAME-RATE=15.000,CODECS="avc1.64000b,mp4a.40.2"
+index-svar1.m3u8
+
+URL: /index-svar1.m3u8
+HEADERS: 200 application/vnd.apple.mpegurl
+BODY: #EXTM3U
+#EXT-X-TARGETDURATION:8
+#EXT-X-VERSION:3
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-DISCONTINUITY-SEQUENCE:0
+#EXT-X-INDEPENDENT-SEGMENTS
+#EXT-X-ALLOW-CACHE:YES
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:00.000+00:00
+#EXTINF:8.342,
+#EXT-X-BITRATE:5124
+seg-1-svar1.ts
+#EXTINF:7.966,
+#EXT-X-BITRATE:16612
+seg-2-svar1.ts
+#EXT-X-DISCONTINUITY
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:16.342+00:00
+#EXTINF:3.200,
+#EXT-X-BITRATE:9369
+seg-3-svar1.ts
+#EXT-X-DISCONTINUITY
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:19.630+00:00
+#EXTINF:3.956,
+#EXT-X-BITRATE:112
+seg-5-svar1.ts
+#EXTINF:3.934,
+#EXT-X-BITRATE:145
+seg-6-svar1.ts
+#EXTINF:1.989,
+#EXT-X-BITRATE:152
+seg-7-svar1.ts
+#EXT-X-ENDLIST
+
+URL: /seg-1-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 5348788, MD5: f120a8a081f82c373c2699c75a46c728
+
+URL: /seg-2-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 16418228, MD5: 7bb081750052c1cb319753d8110c96f5
+
+URL: /seg-3-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 3826740, MD5: 61905a4e992990544087081a6855f90a
+
+URL: /seg-5-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 57716, MD5: 9aba89a2ef46c155e15b0f587bad8a61
+
+URL: /seg-6-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 72756, MD5: 57df738394e340264cde12be20eaedfa
+
+URL: /seg-7-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 39668, MD5: 4abc468def0cb3c8a592da711861fd1c

Original file line number	Diff line number	Diff line change
`@@ -167,7 +167,7 @@ http {`
`167`	`167`	`location /ksmp_proxy/ {`
`168`	`168`	`internal;`
`169`	`169`	`proxy_pass http://127.0.0.1:8001/ksmp/;`
`170`		`- subrequest_output_buffer_size 5m;`
	`170`	`+ subrequest_output_buffer_size 20m;`
`171`	`171`	`}`
`172`	`172`
`173`	`173`	`# curl localhost:8001/sgts/<channel_id>/<bucket_id>/seg-<segment_index>-s<track_int_id>.ts`