Fix video lead cap, hardware frame pool sizing, and decoder packet handling

README.md

@@ -348,6 +348,8 @@ Stats are exposed through OBS's `proc_handler` API under the `get_stats` call, a
 | `obs_lead_ms` | int | How far ahead of real time audio is queued inside OBS (healthy is roughly 60 to 100ms) |
 | `audio_decoder_flushes` | int | Number of audio decoder flushes after repeated decode errors |
 | `video_decoder_flushes` | int | Number of video decoder flushes after repeated decode errors |
+| `video_lead_ms` | int | How far ahead of real time the last video frame was timestamped, before the lead cap. Tracks the audio buffer; a value climbing well past Target Buffer and staying there means concealment has inflated the A/V mapping |
+| `video_lead_clamps` | int | Frames whose lead was capped to keep OBS from dropping its async video queue. Non-zero means video ran ahead of audio rather than freezing |
 | `stream_delay_ms` | int | End-to-end stream delay (SRT latency + decode + buffering) |
 | `low_latency_audio` | bool | Whether OBS async unbuffered low-latency mode is enabled |
 | `reconnect_count` | int | Number of reconnect attempts since the source was created |

deps/versions.env

@@ -6,8 +6,8 @@
 # Bumping a version here also requires updating the matching SHA256.
 
 # Windows only. Linux and macOS use the system zlib; Windows has none.
-ZLIB_VERSION=1.3.1
-ZLIB_SHA256=9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23
+ZLIB_VERSION=1.3.2
+ZLIB_SHA256=bb329a0a2cd0274d05519d61c667c062e06990d72e125ee2dfa8de64f0119d16
 
 FFMPEG_VERSION=9.0
 FFMPEG_SHA256=7f607a00dd0d28a729d5a4811205812eef01cf6ef6155025febb6f36a9062d52
@@ -17,14 +17,14 @@ FFMPEG_SHA256=7f607a00dd0d28a729d5a4811205812eef01cf6ef6155025febb6f36a9062d52
 SRT_VERSION=1.5.6
 SRT_SHA256=2c4980c2c4cfd142d21b829d939dc51db9c6628af5967fff62fd7290769569c7
 
-MBEDTLS_VERSION=3.6.4
-MBEDTLS_SHA256=ec35b18a6c593cf98c3e30db8b98ff93e8940a8c4e690e66b41dfc011d678110
+MBEDTLS_VERSION=3.6.7
+MBEDTLS_SHA256=a7e8bcbec0e6f761b4af24f25677626b35f762f68eef79c08677a363212d11f6
 
 # RIST ingest. obs-deps pins 0.2.7; this is the same story as libsrt.
-LIBRIST_VERSION=0.2.18
-LIBRIST_SHA256=9a2d16dcdb9fb067b7ba4259a3976ff6f8df9a62dbec7f32f19a0b60ec0c114a
+LIBRIST_VERSION=0.2.20
+LIBRIST_SHA256=9e40eeb87f014790531ad41326cc271b930a65962e4b15231b301fc59b29fe31
 
 # Headers only. FFmpeg loads nvcuda/nvcuvid at runtime, so this adds no
 # build-time or load-time dependency on a CUDA install.
-NVCODEC_VERSION=13.0.19.0
-NVCODEC_SHA256=86d15d1a7c0ac73a0eafdfc57bebfeba7da8264595bf531cf4d8db1c22940116
+NVCODEC_VERSION=13.1.15.0
+NVCODEC_SHA256=2255bc74d038b95aa4be30f5f66322c2176acbdb90ada1851db6993536fbeaf7

include/irl-source.h

@@ -90,6 +90,81 @@ struct irl_source;
  * buffer capacity (4x buffer_max_ms) or writes would drop old data. */
 #define IRL_BLEED_PACE_FILL_MS 1000
 
+/* Concealment inflates the audio->OBS playout offset with no bounded
+ * recovery once primed (see irl_audio_maybe_reanchor_offset). This far
+ * past the primed baseline the accumulated latency is treated as
+ * unrecoverable by the speed-drain and reclaimed with one declared
+ * re-anchor. Set above the worst normal buffer swing (buffer_max is
+ * only ~200ms over target) so ordinary adaptive-speed excursions
+ * never trip it; only a real outage's worth of concealment does.
+ *
+ * Lives here rather than next to its use in receiver-audio.c because the
+ * video lead cap below is defined in terms of it. */
+#define AUDIO_OFFSET_REANCHOR_MARGIN_MS 400
+
+/* Video output lead cap.
+ *
+ * libobs schedules async video itself. obs_source_output_video() only queues
+ * the frame; ready_async_frame() releases it once the queue's play head
+ * (last_frame_ts, which advances at wall-clock rate) reaches its timestamp.
+ * Two libobs limits follow from that, and neither is observable from here:
+ *
+ *   - a frame timestamped past the play head holds the *previous* frame on
+ *     screen until wall clock catches up, and
+ *   - at MAX_ASYNC_FRAMES (30) queued frames, cache_video() drops the
+ *     incoming frame, throws the entire queue away and resets last_frame_ts,
+ *     silently.
+ *
+ * Video PTS is mapped through the audio playout offset for lip sync, so the
+ * lead over wall clock tracks the audio jitter buffer, and concealment
+ * inflates it further on every dropout. What libobs actually queues is the
+ * *growth* in that lead since its play head last anchored, so a steady lead
+ * (however large) is free and an excursion is what fills the queue. A backlog
+ * excursion big enough to park 30 frames makes libobs wipe the queue every
+ * time it refills: freeze frames and forward jumps, while audio — which paces
+ * itself — plays clean. That is the shape of the field report this cap exists
+ * for.
+ *
+ * Cap the lead at the configured target buffer plus an excursion allowance.
+ * Only the excursion is cut, never the steady lead, so a deliberately large
+ * Target Buffer still buys real buffered lip sync at any size. Past the cap
+ * the trade is explicit: video runs ahead of audio by the excess instead of
+ * freezing.
+ *
+ * The allowance is bounded from both directions, and both bounds are real:
+ *
+ *   - Above, by IRL_OBS_ASYNC_FRAME_BUDGET frames' worth of the *measured*
+ *     frame interval. What libobs queues is frames, not milliseconds, so the
+ *     same 400ms excursion is 12 frames at 30fps and 24 at 60fps. A fixed
+ *     millisecond allowance would therefore either clamp 30fps streams that
+ *     were never at risk (paying lip sync for nothing) or fail to protect
+ *     60fps ones. The budget leaves headroom under libobs's 30 for the frame
+ *     in flight, tick jitter, and interval mis-estimates.
+ *
+ *   - Below, by AUDIO_OFFSET_REANCHOR_MARGIN_MS. A clamped lead means video
+ *     is early by the excess, which is only acceptable while the excess is
+ *     transient. Excess fill always is: the speed controller drains backlog
+ *     to target at up to +5%, and video re-syncs smoothly on the way back
+ *     because the clamped line and the mapped line meet tangentially. Excess
+ *     *concealment* offset is not — the only thing that ever reclaims it is
+ *     irl_audio_maybe_reanchor_offset(), which does nothing until the drift
+ *     passes that margin. Allow less than it and the band between the two
+ *     becomes permanent lip sync error instead of a transient one.
+ *
+ * The two only conflict above ~60fps, where the frame budget wants less than
+ * the re-anchor margin. The floor wins there — permanent desync is worse than
+ * a queue wipe — and the case is logged, because that is the point where
+ * libobs scheduling has run out and the frames need pacing in-plugin instead.
+ */
+#define IRL_OBS_ASYNC_FRAME_BUDGET 24
+#define IRL_VIDEO_LEAD_WARN_INTERVAL_NS 10000000000ULL
+
+/* Bounds on the measured frame interval (250fps..10fps) and the estimate
+ * used before enough frames have arrived to measure one. */
+#define IRL_VIDEO_INTERVAL_MIN_NS 4000000LL
+#define IRL_VIDEO_INTERVAL_MAX_NS 100000000LL
+#define IRL_VIDEO_INTERVAL_DEFAULT_NS 33333333LL
+
 /* Abort a blocking read/connect through the FFmpeg interrupt callback
  * after this long without progress. A dead-but-open connection (uplink
  * loss in a dead zone) otherwise hangs av_read_frame forever with no
@@ -150,8 +225,9 @@ struct irl_source {
 	 * Decouples the GPU→CPU frame transfer and format conversion
 	 * from the receiver thread so a GPU stall cannot starve audio
 	 * decode. Depth stays small because queued HW frames pin
-	 * decoder surface-pool entries (matched by extra_hw_frames at
-	 * decoder open). Queued frame->pts is in nanoseconds; the
+	 * decoder surface-pool entries (covered by extra_hw_frames at
+	 * decoder open, which budgets this queue plus the two frames
+	 * in flight around it). Queued frame->pts is in nanoseconds; the
 	 * receiver converts before queueing because it may close
 	 * fmt_ctx while frames are still in flight. */
 #define IRL_VIDEO_QUEUE_SIZE 4
@@ -161,6 +237,17 @@ struct irl_source {
 	int video_queue_head;
 	int video_queue_count;
 	uint64_t video_queue_drops;
+	/* Decoder surfaces this plugin pins at once, for checking the
+	 * extra_hw_frames budget against reality rather than against a
+	 * reading of the code: frames sitting in the queue plus the one the
+	 * video thread has popped and is converting. The frame the decoder
+	 * has just handed the receiver thread is not counted (it lives on
+	 * the other thread and is unref'd immediately), so the pool
+	 * requirement is this peak plus one. Cumulative for the source —
+	 * a two-hour stream's high-water mark is the interesting number.
+	 * Both guarded by video_queue_lock. */
+	int video_in_flight;
+	int video_pinned_peak;
 	/* Set by the receiver thread on disconnect, consumed by the video
 	 * thread. Guarded by video_queue_lock. The clear has to run on the
 	 * video thread so it cannot be undone by a frame that was already
@@ -208,6 +295,11 @@ struct irl_source {
 	bool video_ts_init;
 	uint64_t video_sys_base;  /* os_gettime_ns() at first frame */
 	int64_t video_pts_base;   /* stream PTS at first frame (in ns) */
+	/* Previous decoded PTS, receiver-thread-owned; feeds the frame
+	 * interval EMA. */
+	int64_t video_prev_pts_ns;
+	/* Throttle for the lead-cap warning, video-thread-owned. */
+	uint64_t video_lead_warn_time_ns;
 
 	/* Audio output clock.  OBS timestamps are a pure sample
 	 * counter anchored once at prime time:
@@ -252,6 +344,19 @@ struct irl_source {
 	int64_t latest_audio_stream_pts_ns;
 	int64_t latest_video_stream_pts_ns;
 
+	/* Video lead diagnostics, all guarded by audio_state_lock.
+	 *
+	 * video_frame_interval_ns is an EMA of decoded PTS deltas, written
+	 * by the receiver thread and read by the video thread to estimate
+	 * how many frames a given lead parks in the libobs async queue.
+	 * video_lead_ns is the lead the PTS mapping asked for before the
+	 * cap (the uncapped value is the diagnostic: it shows the ratchet),
+	 * written by the video thread and read by the OBS thread. */
+	int64_t video_frame_interval_ns;
+	int64_t video_lead_ns;
+	int64_t video_lead_peak_ns;
+	uint64_t video_lead_clamps;
+
 	/* Latest audio already queued to OBS, in OBS clock domain.
 	 * Used to align video to actual audio playout instead of
 	 * approximating from the plugin-side jitter-buffer fill. */
@@ -297,6 +402,22 @@ struct irl_source {
 	 * not reset the decoder state (losing reference frames). */
 	int audio_decode_errors;
 	int video_decode_errors;
+	/* Jitter-buffer high-water mark, same sampling argument as
+	 * video_lead_peak_ns: the backlog excursion that drives everything
+	 * here is transient, and `buf` at log time usually misses it. */
+	int audio_fill_peak_ms;
+	/* avcodec_send_packet() returned EAGAIN, meaning the decoder did not
+	 * accept the packet and it must be resent after draining output.
+	 * Rare when the frame pool is adequately sized, which is exactly why
+	 * a non-zero count is worth seeing: it is the signal that decoder
+	 * surfaces are exhausted. */
+	uint64_t video_pkt_eagain;
+	uint64_t audio_pkt_eagain;
+	/* Packets still refused after the drain-and-resend retry, and so
+	 * genuinely lost. This is the number that costs picture quality;
+	 * the eagain counters above only say the condition was hit. */
+	uint64_t video_pkt_dropped;
+	uint64_t audio_pkt_dropped;
 	uint64_t audio_decoder_flushes;
 	uint64_t video_decoder_flushes;
 	uint64_t audio_last_decoder_flush_time_us;

src/irl-source.c

@@ -325,6 +325,8 @@ static void irl_source_get_stats(void *data, calldata_t *cd)
 	uint64_t video_sys_base = ctx->video_sys_base;
 	int64_t video_pts_base = ctx->video_pts_base;
 	int64_t latest_video_stream_pts_ns = ctx->latest_video_stream_pts_ns;
+	int64_t video_lead_ms = ctx->video_lead_ns / 1000000LL;
+	uint64_t video_lead_clamps = ctx->video_lead_clamps;
 	irl_mutex_unlock(&ctx->audio_state_lock);
 
 	calldata_set_int(cd, "buffer_fill_ms", buffer_fill_ms);
@@ -362,6 +364,9 @@ static void irl_source_get_stats(void *data, calldata_t *cd)
 			 (long long)audio_decoder_flushes);
 	calldata_set_int(cd, "video_decoder_flushes",
 			 (long long)video_decoder_flushes);
+	calldata_set_int(cd, "video_lead_ms", (long long)video_lead_ms);
+	calldata_set_int(cd, "video_lead_clamps",
+			 (long long)video_lead_clamps);
 
 	/* Stream delay: how far behind real-time the video output is.
 	 * Computed as wall_clock - anchored_video_PTS.  Includes SRT
@@ -429,6 +434,7 @@ void *irl_source_create(obs_data_t *settings, obs_source_t *source)
 		"out int audio_output_restarts, out int obs_lead_ms, "
 		"out int audio_decoder_flushes, "
 		"out int video_decoder_flushes, "
+		"out int video_lead_ms, out int video_lead_clamps, "
 		"out int stream_delay_ms, out bool low_latency_audio, "
 		"out int reconnect_count)",
 		irl_source_get_stats, ctx);

src/receiver-audio.c

@@ -58,14 +58,11 @@
  * the clock line instead of letting OBS add permanent buffering. */
 #define AUDIO_OUT_MAX_LAG_MS 150
 
-/* Concealment inflates the audio->OBS playout offset with no bounded
- * recovery once primed (see the offset-reanchor logic below). This
- * far past the primed baseline the accumulated latency is treated as
- * unrecoverable by the speed-drain and reclaimed with one declared
- * re-anchor. Set above the worst normal buffer swing (buffer_max is
- * only ~200ms over target) so ordinary adaptive-speed excursions
- * never trip it; only a real outage's worth of concealment does. */
-#define AUDIO_OFFSET_REANCHOR_MARGIN_MS 400
+/* AUDIO_OFFSET_REANCHOR_MARGIN_MS lives in irl-source.h: it is also the
+ * floor on the video lead cap's excursion allowance, because a lead the cap
+ * suppresses and this margin never reclaims is permanent lip sync error
+ * rather than transient. See the IRL_OBS_ASYNC_FRAME_BUDGET block there
+ * before changing it. */
 
 /* Playback speed authority for buffer regulation. Asymmetric,
  * IRLToolkit-style: draining a post-stall backlog runs up to +5%
@@ -134,6 +131,13 @@ void irl_reset_stream_timing_state(struct irl_source *ctx)
 	irl_reset_audio_timing_state(ctx);
 	ctx->video_ts_init = false;
 	ctx->latest_video_stream_pts_ns = 0;
+	/* State, not counters: the interval has to be re-measured for the
+	 * new stream, and a stale lead would be reported until the first
+	 * frame arrives. video_lead_clamps is cumulative for the source,
+	 * like the other quality counters. */
+	ctx->video_prev_pts_ns = 0;
+	ctx->video_frame_interval_ns = 0;
+	ctx->video_lead_ns = 0;
 	ctx->video_decode_errors = 0;
 	ctx->video_last_decoder_flush_time_us = 0;
 	ctx->video_last_decoder_warning_time_us = 0;
@@ -693,6 +697,8 @@ bool irl_pump_audio_once(struct irl_source *ctx)
 	int chunk_count = 0;
 	bool has_audio = audio_buffer_peek_state(&ctx->audio_buf, &peek,
 						 &fill_ms, &chunk_count);
+	if (fill_ms > ctx->audio_fill_peak_ms)
+		ctx->audio_fill_peak_ms = fill_ms;
 
 	if (has_audio &&
 	    maybe_trim_hidden_audio_backlog(ctx, fill_ms, chunk_count))