ngx-live: fix audio ends before video issue

erankor · erankor · commit 02f40103fbf5 · 2024-06-01T19:40:01.000+03:00
the segmenter may try to cut a segment at the audio end timestamp, there
is no video key frame at this timestamp - the video segments come out
empty, and some of the audio frames are discarded.
then it tries to create another segment, but the segments of all tracks
come out empty, and the channel is failed.
the fix is to remove all pending frames in instead of failing, if all
tracks are inactive
diff --git a/nginx-live-module/src/ngx_live_segmenter.c b/nginx-live-module/src/ngx_live_segmenter.c
@@ -1986,7 +1986,8 @@ ngx_live_segmenter_remove_frames(ngx_live_track_t *track, ngx_uint_t count,
 
 
 static void
-ngx_live_segmenter_remove_all_frames(ngx_live_track_t *track)
+ngx_live_segmenter_remove_all_frames(ngx_live_track_t *track,
+    ngx_flag_t reconnect)
 {
     ngx_live_segmenter_track_ctx_t  *ctx;
 
@@ -1995,7 +1996,7 @@ ngx_live_segmenter_remove_all_frames(ngx_live_track_t *track)
         return;
     }
 
-    if (ctx->frame_count > 0) {
+    if (ctx->frame_count > 0 && reconnect) {
         (void) ngx_live_core_track_event(track,
             NGX_LIVE_EVENT_TRACK_RECONNECT, NULL);
     }
@@ -2022,6 +2023,23 @@ ngx_live_segmenter_remove_all_frames(ngx_live_track_t *track)
 }
 
 
+static void
+ngx_live_segmenter_remove_all_channel_frames(ngx_live_channel_t *channel)
+{
+    ngx_queue_t       *q;
+    ngx_live_track_t  *cur_track;
+
+    for (q = ngx_queue_head(&channel->tracks.queue);
+        q != ngx_queue_sentinel(&channel->tracks.queue);
+        q = ngx_queue_next(q))
+    {
+        cur_track = ngx_queue_data(q, ngx_live_track_t, queue);
+
+        ngx_live_segmenter_remove_all_frames(cur_track, 0);
+    }
+}
+
+
 static void
 ngx_live_segmenter_prepare_create_segment(ngx_live_channel_t *channel,
     uint32_t *media_types_mask, int64_t *min_pts, int64_t *min_created)
@@ -2605,13 +2623,15 @@ ngx_live_segmenter_dispose_segment(ngx_live_channel_t *channel,
     int64_t end_pts)
 {
     ngx_flag_t                         removed;
+    ngx_flag_t                         active_tracks;
     ngx_flag_t                         force_new_period;
     ngx_queue_t                       *q;
     ngx_live_track_t                  *cur_track;
     ngx_live_segmenter_track_ctx_t    *cur_ctx;
     ngx_live_segmenter_channel_ctx_t  *cctx;
 
     removed = 0;
+    active_tracks = 0;
     force_new_period = 0;
 
     for (q = ngx_queue_head(&channel->tracks.queue);
@@ -2623,6 +2643,12 @@ ngx_live_segmenter_dispose_segment(ngx_live_channel_t *channel,
             ngx_live_segmenter_module);
 
         if (cur_ctx->copy_index <= 0) {
+            if (cur_ctx->frame_count > 0
+                && cur_ctx->state != ngx_live_track_inactive)
+            {
+                active_tracks = 1;
+            }
+
             continue;
         }
 
@@ -2642,6 +2668,14 @@ ngx_live_segmenter_dispose_segment(ngx_live_channel_t *channel,
     }
 
     if (!removed) {
+        if (!active_tracks) {
+            ngx_log_error(NGX_LOG_INFO, &channel->log, 0,
+                "ngx_live_segmenter_dispose_segment: "
+                "no active tracks, removing all frames");
+            ngx_live_segmenter_remove_all_channel_frames(channel);
+            return NGX_OK;
+        }
+
         ngx_log_error(NGX_LOG_ALERT, &channel->log, 0,
             "ngx_live_segmenter_dispose_segment: no frames removed");
         return NGX_ERROR;
@@ -3319,7 +3353,7 @@ ngx_live_segmenter_start_stream(ngx_live_stream_stream_req_t *req)
 
     track = req->track;
 
-    ngx_live_segmenter_remove_all_frames(track);
+    ngx_live_segmenter_remove_all_frames(track, 1);
 
     initial_frame_id = req->header->c.initial_frame_id;
     if (track->next_frame_id > initial_frame_id) {
diff --git a/nginx-live-module/test/nginx.conf b/nginx-live-module/test/nginx.conf
@@ -167,7 +167,7 @@ http {
         location /ksmp_proxy/ {
             internal;
             proxy_pass http://127.0.0.1:8001/ksmp/;
-            subrequest_output_buffer_size 5m;
+            subrequest_output_buffer_size 20m;
         }
 
         # curl localhost:8001/sgts/<channel_id>/<bucket_id>/seg-<segment_index>-s<track_int_id>.ts
diff --git a/nginx-live-module/test/tests/audio_ends_before_video.py b/nginx-live-module/test/tests/audio_ends_before_video.py
@@ -0,0 +1,49 @@
+from test_base import *
+
+# EXPECTED:
+#   16 sec first video
+#   10 sec second video
+
+def test(channelId=CHANNEL_ID):
+    st = KmpSendTimestamps()
+
+    nl = setupChannelTimeline(channelId)
+
+    rv = KmpMediaFileReader(TEST_VIDEO_HIGH, 0)
+    ra = KmpMediaFileReader(TEST_VIDEO_HIGH, 1)
+
+    sv, sa = createVariant(nl, VARIANT_ID, [('v1', 'video'), ('a1', 'audio')])
+
+    kmpSendStreams([
+        (rv, sv),
+        (ra, sa),
+    ], st, 17.5, realtime=5)
+
+    kmpSendStreams([
+        (rv, sv),
+        (ra, KmpNullSender()),
+    ], st, 2, realtime=5)
+
+    kmpSendEndOfStream([sv, sa])
+
+    time.sleep(2)
+
+    initialFrameId = 1000000
+    
+    rv = KmpMediaFileReader(TEST_VIDEO1, 0)
+    ra = KmpMediaFileReader(TEST_VIDEO1, 1)
+
+    sv, sa = createVariant(nl, VARIANT_ID, [('v1', 'video'), ('a1', 'audio')], initialFrameId=initialFrameId)
+
+    kmpSendStreams([
+        (rv, sv),
+        (ra, sa),
+    ], st, 10, realtime=5)
+
+    kmpSendEndOfStream([sv, sa])
+
+    # deactivate the timeline
+    nl.timeline.update(NginxLiveTimeline(id=TIMELINE_ID, end_list='on'))
+
+    testDefaultStreams(channelId, __file__)
+    logTracker.assertContains(b'no active tracks, removing all frames')
diff --git a/nginx-live-module/test/tests/ref/audio_ends_before_video-hls-fmp4.txt b/nginx-live-module/test/tests/ref/audio_ends_before_video-hls-fmp4.txt
@@ -0,0 +1,66 @@
+URL: /master.m3u8
+HEADERS: 200 application/vnd.apple.mpegurl
+BODY: #EXTM3U
+#EXT-X-INDEPENDENT-SEGMENTS
+
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=126573,AVERAGE-BANDWIDTH=114074,RESOLUTION=160x120,FRAME-RATE=15.000,CODECS="avc1.64000b,mp4a.40.2"
+index-svar1.m3u8
+
+URL: /index-svar1.m3u8
+HEADERS: 200 application/vnd.apple.mpegurl
+BODY: #EXTM3U
+#EXT-X-TARGETDURATION:8
+#EXT-X-VERSION:6
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-DISCONTINUITY-SEQUENCE:0
+#EXT-X-INDEPENDENT-SEGMENTS
+#EXT-X-ALLOW-CACHE:YES
+#EXT-X-MAP:URI="init-1-svar1.mp4"
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:00.000+00:00
+#EXTINF:8.342,
+#EXT-X-BITRATE:4971
+seg-1-svar1.m4s
+#EXTINF:7.966,
+#EXT-X-BITRATE:16215
+seg-2-svar1.m4s
+#EXT-X-DISCONTINUITY
+#EXT-X-MAP:URI="init-3-svar1.mp4"
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:19.630+00:00
+#EXTINF:3.956,
+#EXT-X-BITRATE:92
+seg-3-svar1.m4s
+#EXTINF:3.934,
+#EXT-X-BITRATE:123
+seg-4-svar1.m4s
+#EXTINF:1.989,
+#EXT-X-BITRATE:124
+seg-5-svar1.m4s
+#EXT-X-ENDLIST
+
+URL: /init-1-svar1.mp4
+HEADERS: 200 video/mp4
+BODY: SIZE: 1106, MD5: 44623fdc2bb6eb1183e22c022be3e765
+
+URL: /seg-1-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 5180236, MD5: 0d015341c4fbb773cfcc90ecbce81d0d
+
+URL: /seg-2-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 16014394, MD5: 309376ada6e59c0222bf0bab74028e1e
+
+URL: /init-3-svar1.mp4
+HEADERS: 200 video/mp4
+BODY: SIZE: 1106, MD5: ca1c220b1d37d697270c21bce220cdef
+
+URL: /seg-3-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 46063, MD5: 1a0e28466ca0bbefea4e4a2764349bb5
+
+URL: /seg-4-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 61926, MD5: 3bfcec2fa6c0001e0bd632116c54cb87
+
+URL: /seg-5-svar1.m4s
+HEADERS: 200 video/mp4
+BODY: SIZE: 32681, MD5: 6c15984785f28b71b14288e1feb3ddcf
diff --git a/nginx-live-module/test/tests/ref/audio_ends_before_video-hls-ts.txt b/nginx-live-module/test/tests/ref/audio_ends_before_video-hls-ts.txt
@@ -0,0 +1,56 @@
+URL: /master.m3u8
+HEADERS: 200 application/vnd.apple.mpegurl
+BODY: #EXTM3U
+#EXT-X-INDEPENDENT-SEGMENTS
+
+#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=144154,AVERAGE-BANDWIDTH=131383,RESOLUTION=160x120,FRAME-RATE=15.000,CODECS="avc1.64000b,mp4a.40.2"
+index-svar1.m3u8
+
+URL: /index-svar1.m3u8
+HEADERS: 200 application/vnd.apple.mpegurl
+BODY: #EXTM3U
+#EXT-X-TARGETDURATION:8
+#EXT-X-VERSION:3
+#EXT-X-MEDIA-SEQUENCE:0
+#EXT-X-DISCONTINUITY-SEQUENCE:0
+#EXT-X-INDEPENDENT-SEGMENTS
+#EXT-X-ALLOW-CACHE:YES
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:00.000+00:00
+#EXTINF:8.342,
+#EXT-X-BITRATE:5124
+seg-1-svar1.ts
+#EXTINF:7.966,
+#EXT-X-BITRATE:16612
+seg-2-svar1.ts
+#EXT-X-DISCONTINUITY
+#EXT-X-PROGRAM-DATE-TIME:2020-01-01T00:00:19.630+00:00
+#EXTINF:3.956,
+#EXT-X-BITRATE:112
+seg-3-svar1.ts
+#EXTINF:3.934,
+#EXT-X-BITRATE:145
+seg-4-svar1.ts
+#EXTINF:1.989,
+#EXT-X-BITRATE:152
+seg-5-svar1.ts
+#EXT-X-ENDLIST
+
+URL: /seg-1-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 5348788, MD5: f120a8a081f82c373c2699c75a46c728
+
+URL: /seg-2-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 16418228, MD5: 7bb081750052c1cb319753d8110c96f5
+
+URL: /seg-3-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 57716, MD5: a959bb97cc3958bc597e4a0bae619595
+
+URL: /seg-4-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 72756, MD5: 4c2639257ffcf8a277213de890179e89
+
+URL: /seg-5-svar1.ts
+HEADERS: 200 video/mp2t
+BODY: SIZE: 39668, MD5: 4d3f7adfdf83235fa0a67f85c6b68f3d

Original file line number	Diff line number	Diff line change
`@@ -167,7 +167,7 @@ http {`
`167`	`167`	`location /ksmp_proxy/ {`
`168`	`168`	`internal;`
`169`	`169`	`proxy_pass http://127.0.0.1:8001/ksmp/;`
`170`		`- subrequest_output_buffer_size 5m;`
	`170`	`+ subrequest_output_buffer_size 20m;`
`171`	`171`	`}`
`172`	`172`
`173`	`173`	`# curl localhost:8001/sgts/<channel_id>/<bucket_id>/seg-<segment_index>-s<track_int_id>.ts`