ffmpeg: Unit tsts for short segs. Fix filter flush

- Use expected output pkt pts in flush packet to ensure fps filter returns a frame even for 1-frame segments getting downsampled to low fps. FFmpeg CLI drops such frames unless using eof_action=pass, but we cannot use it as it closes the decoder. - Add parameterized unit tests to reproduce #168 Fixes #168
livepeer · May 12, 2020 · 13e2a17 · 13e2a17
1 parent e27e06e
commit 13e2a17
Show file tree

Hide file tree

Showing 3 changed files with 188 additions and 7 deletions.
diff --git a/ffmpeg/api_test.go b/ffmpeg/api_test.go
@@ -478,3 +478,168 @@ func TestTranscoder_API_AlternatingTimestamps(t *testing.T) {
   `
 	run(cmd)
 }
+
+// test short segments
+func shortSegments(t *testing.T, accel Acceleration, fc int) {
+	run, dir := setupTest(t)
+	defer os.RemoveAll(dir)
+
+	cmd := `
+    # generate segments with #fc frames
+    cp "$1/../transcoder/test.ts" .
+	frame_count=%d
+
+    ffmpeg -loglevel warning -ss 0 -i test.ts -c copy -frames:v $frame_count -copyts short0.ts
+    ffmpeg -loglevel warning -ss 2 -i test.ts -c copy -frames:v $frame_count -copyts short1.ts
+    ffmpeg -loglevel warning -ss 4 -i test.ts -c copy -frames:v $frame_count -copyts short2.ts
+    ffmpeg -loglevel warning -ss 6 -i test.ts -c copy -frames:v $frame_count -copyts short3.ts
+
+    ffprobe -loglevel warning -count_frames -show_streams -select_streams v short0.ts | grep nb_read_frames=$frame_count
+    ffprobe -loglevel warning -count_frames -show_streams -select_streams v short1.ts | grep nb_read_frames=$frame_count
+    ffprobe -loglevel warning -count_frames -show_streams -select_streams v short2.ts | grep nb_read_frames=$frame_count
+    ffprobe -loglevel warning -count_frames -show_streams -select_streams v short3.ts | grep nb_read_frames=$frame_count
+  `
+	run(fmt.Sprintf(cmd, fc))
+
+	// Test if decoding/encoding expected number of frames
+	tc := NewTranscoder()
+	defer tc.StopTranscoder()
+	for i := 0; i < 4; i++ {
+		fname := fmt.Sprintf("%s/short%d.ts", dir, i)
+		t.Log("fname ", fname)
+		in := &TranscodeOptionsIn{Fname: fname, Accel: accel}
+		out := []TranscodeOptions{{Oname: dir + "/out.ts", Profile: P144p30fps16x9, Accel: accel}}
+		res, err := tc.Transcode(in, out)
+		if err != nil {
+			t.Error(err)
+		}
+		if fc != res.Decoded.Frames {
+			t.Error("Did not decode expected number of frames: ", res.Decoded.Frames)
+		}
+		if 0 == res.Encoded[0].Frames {
+			// not sure what should be a reasonable number here
+			t.Error("Did not encode any frames: ", res.Encoded[0].Frames)
+		}
+	}
+
+	// test standalone stream copy
+	tc.StopTranscoder()
+	tc = NewTranscoder()
+	for i := 0; i < 4; i++ {
+		fname := fmt.Sprintf("%s/short%d.ts", dir, i)
+		oname := fmt.Sprintf("%s/vcopy%d.ts", dir, i)
+		in := &TranscodeOptionsIn{Fname: fname, Accel: accel}
+		out := []TranscodeOptions{
+			{
+				Oname: oname,
+				VideoEncoder: ComponentOptions{Name: "copy", Opts: map[string]string{
+					"mpegts_flags": "resend_headers,initial_discontinuity",
+				}},
+				Accel: accel,
+			},
+		}
+		res, err := tc.Transcode(in, out)
+		if err != nil {
+			t.Error(err)
+		}
+		if res.Encoded[0].Frames != 0 {
+			t.Error("Unexpected frame counts from stream copy")
+			t.Error(res)
+		}
+		cmd = `
+        # extract video track, compare md5sums
+		i=%d
+        ffmpeg -i short$i.ts -an -c:v copy -f md5 short$i.md5
+        ffmpeg -i vcopy$i.ts -an -c:v copy -f md5 vcopy$i.md5
+        diff -u short$i.md5 vcopy$i.md5
+        `
+		run(fmt.Sprintf(cmd, i))
+	}
+
+	// test standalone stream drop
+	tc.StopTranscoder()
+	tc = NewTranscoder()
+	for i := 0; i < 4; i++ {
+		fname := fmt.Sprintf("%s/short%d.ts", dir, i)
+		oname := fmt.Sprintf("%s/vdrop%d.ts", dir, i)
+		// Normal case : drop only video
+		in := &TranscodeOptionsIn{Fname: fname, Accel: accel}
+		out := []TranscodeOptions{
+			{
+				Oname:        oname,
+				VideoEncoder: ComponentOptions{Name: "drop"},
+				Accel:        accel,
+			},
+		}
+		res, err := tc.Transcode(in, out)
+		if err != nil {
+			t.Error(err)
+		}
+		if res.Decoded.Frames != 0 || res.Encoded[0].Frames != 0 {
+			t.Error("Unexpected count of decoded frames ", res.Decoded.Frames, res.Decoded.Pixels)
+		}
+
+	}
+
+	// test framerate passthrough
+	tc.StopTranscoder()
+	tc = NewTranscoder()
+	for i := 0; i < 4; i++ {
+		fname := fmt.Sprintf("%s/short%d.ts", dir, i)
+		oname := fmt.Sprintf("%s/vpassthru%d.ts", dir, i)
+		out := []TranscodeOptions{{Profile: P144p30fps16x9, Accel: accel}}
+		out[0].Profile.Framerate = 0 // Passthrough!
+
+		out[0].Oname = oname
+		in := &TranscodeOptionsIn{Fname: fname, Accel: accel}
+		res, err := tc.Transcode(in, out)
+		if err != nil {
+			t.Error("Could not transcode: ", err)
+		}
+		// verify that output frame count is same as input frame count
+		if res.Decoded.Frames != fc || res.Encoded[0].Frames != fc {
+			t.Error("Did not get expected frame count; got ", res.Encoded[0].Frames)
+		}
+	}
+
+	// test low fps (25) to low fps (20)
+	tc.StopTranscoder()
+	tc = NewTranscoder()
+
+	cmd = `
+    # convert segment to 25fps
+	frame_count=%d
+	ffmpeg -loglevel warning -i test.ts -vf fps=25 -c:v libx264 -c:a copy test25.ts
+
+	# create short segment with #fc frames
+    ffmpeg -loglevel warning -ss 0 -i test25.ts -c copy -frames:v $frame_count -copyts short25.ts
+
+	# sanity check
+	ffprobe -loglevel warning -show_streams short25.ts | grep r_frame_rate=25/1
+    ffprobe -loglevel warning -count_frames -show_streams -select_streams v short25.ts | grep nb_read_frames=$frame_count
+  `
+	run(fmt.Sprintf(cmd, fc))
+
+	fname := fmt.Sprintf("%s/short25.ts", dir)
+	t.Log("fname ", fname)
+	in := &TranscodeOptionsIn{Fname: fname, Accel: accel}
+	out := []TranscodeOptions{{Oname: dir + "/out25.ts", Profile: P144p30fps16x9, Accel: accel}}
+	out[0].Profile.Framerate = 20 // Force 20fps
+	res, err := tc.Transcode(in, out)
+	if err != nil {
+		t.Error(err)
+	}
+	if fc != res.Decoded.Frames {
+		t.Error("Did not decode expected number of frames: ", res.Decoded.Frames)
+	}
+	if 0 == res.Encoded[0].Frames {
+		t.Error("Did not encode any frames: ", res.Encoded[0].Frames)
+	}
+}
+
+func TestTranscoder_ShortSegments(t *testing.T) {
+	shortSegments(t, Software, 1)
+	shortSegments(t, Software, 2)
+	shortSegments(t, Software, 3)
+	shortSegments(t, Software, 5)
+}
diff --git a/ffmpeg/lpms_ffmpeg.c b/ffmpeg/lpms_ffmpeg.c
@@ -976,7 +976,7 @@ static int lpms_receive_frame(struct input_ctx *ictx, AVCodecContext *dec, AVFra
 {
     int ret = avcodec_receive_frame(dec, frame);
     if (dec != ictx->vc) return ret;
-    if (!ret && frame && frame->pts != -1) {
+    if (!ret && frame && !is_flush_frame(frame)) {
       ictx->pkt_diff--; // decrease buffer count for non-sentinel video frames
       if (ictx->flushing) ictx->sentinel_count = 0;
     }
@@ -1059,9 +1059,12 @@ int process_in(struct input_ctx *ictx, AVFrame *frame, AVPacket *pkt)
     ret = lpms_receive_frame(ictx, ictx->vc, frame);
     pkt->stream_index = ictx->vi;
     // Keep flushing if we haven't received all frames back but stop after SENTINEL_MAX tries.
-    if (ictx->pkt_diff != 0 && ictx->sentinel_count <= SENTINEL_MAX) return 0;
-    ictx->flushed = 1;
-    if (!ret) return ret;
+    if (ictx->pkt_diff != 0 && ictx->sentinel_count <= SENTINEL_MAX && (!ret || ret == AVERROR(EAGAIN))) {
+        return 0; // ignore actual return value and keep flushing
+    } else {
+        ictx->flushed = 1;
+        if (!ret) return ret;
+    }
   }
   // Flush audio decoder.
   if (ictx->ac) {
@@ -1177,11 +1180,17 @@ int process_out(struct input_ctx *ictx, struct output_ctx *octx, AVCodecContext
   // Start filter flushing process if necessary
   if (!inf && !filter->flushed) {
     // Set input frame to the last frame
-    // And increment pts offset by pkt_duration
-    // TODO It may make sense to use the expected output packet duration instead
     int is_video = AVMEDIA_TYPE_VIDEO == ost->codecpar->codec_type;
     AVFrame *frame = is_video ? ictx->last_frame_v : ictx->last_frame_a;
-    filter->flush_offset += frame->pkt_duration;
+    // Increment pts offset by:
+    // - input packet's duration usually
+    int64_t dur = frame->pkt_duration;
+    // - output packet's duration if using fps filter & haven't encoded anything yet
+    if (is_video && octx->fps.den && !octx->res->frames) {
+      AVStream *ist = ictx->ic->streams[ictx->vi];
+      dur = av_rescale_q(frame->pkt_duration, ist->r_frame_rate, octx->fps);
+    }
+    filter->flush_offset += dur;
     inf = frame;
     inf->opaque = (void*)inf->pts; // value doesn't matter; just needs to be set
     is_flushing = 1;

diff --git a/ffmpeg/nvidia_test.go b/ffmpeg/nvidia_test.go
@@ -764,4 +764,11 @@ func TestNvidia_API_AlternatingTimestamps(t *testing.T) {
 	tc.StopTranscoder()
 }
 
+func TestNvidia_ShortSegments(t *testing.T) {
+	shortSegments(t, Nvidia, 1)
+	shortSegments(t, Nvidia, 2)
+	shortSegments(t, Nvidia, 3)
+	shortSegments(t, Nvidia, 5)
+}
+
 // XXX test bframes or delayed frames