diff --git a/lib/ex_webrtc/peer_connection/configuration.ex b/lib/ex_webrtc/peer_connection/configuration.ex index a5e492d6..7f668d5c 100644 --- a/lib/ex_webrtc/peer_connection/configuration.ex +++ b/lib/ex_webrtc/peer_connection/configuration.ex @@ -27,31 +27,7 @@ defmodule ExWebRTC.PeerConnection.Configuration do clock_rate: 90_000 } - # Ensure we are using H264 with packetization_mode=1 by default - # (packetization_mode=0 has issues when switching layers) - @default_codec_params_h264 %RTPCodecParameters{ - payload_type: 99, - mime_type: "video/H264", - clock_rate: 90_000, - sdp_fmtp_line: %FMTP{ - pt: 99, - level_asymmetry_allowed: true, - packetization_mode: 1, - profile_level_id: 0x42E01F - } - } - - @default_codec_params_av1 %RTPCodecParameters{ - payload_type: 45, - mime_type: "video/AV1", - clock_rate: 90_000, - sdp_fmtp_line: %FMTP{pt: 45, level_idx: 5, profile: 0, tier: 0} - } - - @default_audio_codecs [@default_codec_params_opus] - - @default_video_codecs [ - @default_codec_params_vp8, + @default_codec_params_h264 [ %RTPCodecParameters{ payload_type: 98, mime_type: "video/H264", @@ -63,10 +39,34 @@ defmodule ExWebRTC.PeerConnection.Configuration do profile_level_id: 0x42E01F } }, - @default_codec_params_h264, - @default_codec_params_av1 + %RTPCodecParameters{ + payload_type: 99, + mime_type: "video/H264", + clock_rate: 90_000, + sdp_fmtp_line: %FMTP{ + pt: 99, + level_asymmetry_allowed: true, + packetization_mode: 1, + profile_level_id: 0x42E01F + } + } ] + @default_codec_params_av1 %RTPCodecParameters{ + payload_type: 45, + mime_type: "video/AV1", + clock_rate: 90_000, + sdp_fmtp_line: %FMTP{pt: 45, level_idx: 5, profile: 0, tier: 0} + } + + @default_audio_codecs [@default_codec_params_opus] + @default_video_codecs [ + @default_codec_params_vp8, + @default_codec_params_h264, + @default_codec_params_av1 + ] + |> List.flatten() + @typedoc """ Allowed audio codec names which will get expanded to the relevant default `t:ExWebRTC.RTPCodecParameters.t/0` """ @@ -718,7 +718,7 @@ defmodule ExWebRTC.PeerConnection.Configuration do options Enum.all?(codecs, &is_atom/1) -> - expanded_codecs = Enum.map(codecs, &expand_default_codec/1) + expanded_codecs = codecs |> Enum.map(&expand_default_codec/1) |> List.flatten() Keyword.put(options, key, expanded_codecs) true -> diff --git a/lib/ex_webrtc/rtp/h264.ex b/lib/ex_webrtc/rtp/h264.ex index 453e4d50..0b5be002 100644 --- a/lib/ex_webrtc/rtp/h264.ex +++ b/lib/ex_webrtc/rtp/h264.ex @@ -7,10 +7,19 @@ defmodule ExWebRTC.RTP.H264 do # Copied nearly 1-to-1 from https://github.com/membraneframework/membrane_rtp_h264_plugin/blob/master/lib/rtp_h264/utils.ex # originally based on galene's implementation https://github.com/jech/galene/blob/6fbdf0eab2c9640e673d9f9ec0331da24cbf2c4c/codecs/codecs.go#L119 - # but only looks for SPS - # it is also unclear why we sometimes check against nalu type == 7 - # and sometimes against nalu type == 5 but galene does it this way - # and it works + # but only looks for SPS, in packetization_mode=0 as well as packetization_mode=1. + # + # It's been empirically tested with simulated packet loss that for packetization_mode=0 (`nalu_type in 1..23` clause): + # * if we're checking against `nalu_type == 5`, the stream breaks regularly when switching layers, + # * if we're checking against `nalu_type == 5 or nalu_type == 7`, the stream breaks occasionally when switching layers, + # this happens when we've lost the packet containing SPS, but received the following one containing the keyframe, + # * if we're checking against `nalu_type == 7`, no issues were encountered. + # + # Janus also does it this way. + # https://github.com/meetecho/janus-gateway/blob/3367f41de9225daed812ca0991c259f1458fe49f/src/utils.h#L352 + # + # For more info, refer to the H264 spec and RFC 6184, sections 5.4 and 6 + # https://datatracker.ietf.org/doc/html/rfc6184#section-5.4 @doc """ Returns a boolean telling if the packets contains a beginning of a H264 intra-frame. @@ -21,14 +30,21 @@ defmodule ExWebRTC.RTP.H264 do def keyframe?(%Packet{}), do: false + # Reserved defp do_keyframe?(0, _), do: false - defp do_keyframe?(nalu_type, _) when nalu_type in 1..23, do: nalu_type == 5 + + # Single NAL Unit packets: check if NALU contains SPS (type 7) + defp do_keyframe?(nalu_type, _) when nalu_type in 1..23, do: nalu_type == 7 + + # STAP-A defp do_keyframe?(24, aus), do: check_aggr_units(24, aus) + # STAP-B, MTAP16, MTAP24 defp do_keyframe?(nalu_type, <<_don::16, aus::binary>>) when nalu_type in 25..27, do: check_aggr_units(nalu_type, aus) + # FU-A, FU-B defp do_keyframe?(nalu_type, <>) when nalu_type in 28..29, do: s == 1 and type == 7 diff --git a/test/ex_webrtc/peer_connection/configuration_test.exs b/test/ex_webrtc/peer_connection/configuration_test.exs index 9db03745..6c3bb9a5 100644 --- a/test/ex_webrtc/peer_connection/configuration_test.exs +++ b/test/ex_webrtc/peer_connection/configuration_test.exs @@ -515,11 +515,13 @@ defmodule ExWebRTC.PeerConnection.ConfigurationTest do assert Configuration.expand_default_codecs(og_options) == og_options - [video_codecs: [vp8_params, h264_params]] = + # 2 packetization_modes for H264 + [video_codecs: [vp8_params, h264_params_0, h264_params_1]] = Configuration.expand_default_codecs(video_codecs: [:vp8, :h264]) assert vp8_params.mime_type == "video/VP8" - assert h264_params.mime_type == "video/H264" + assert h264_params_0.mime_type == "video/H264" + assert h264_params_1.mime_type == "video/H264" assert_raise RuntimeError, fn -> Configuration.expand_default_codecs(video_codecs: [:av2]) end