Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support AV1-specific media format parameters #619

Merged
merged 2 commits into from
Feb 2, 2025
Merged

Support AV1-specific media format parameters #619

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
055301d
add av1 format params
alexlapa Feb 2, 2025
138808c
changelog
alexlapa Feb 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
* Limit TWCC iteration with packet status count #606
* Dedupe acked packets from `TwccSendRegister::apply_report()` #601, #605
* Align BWE ArrivalGroup calculation with libwebrtc implementation #608, #615
* Support AV1-specific media format parameters #619

# 0.6.2

Expand Down
81 changes: 77 additions & 4 deletions src/format.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,35 @@ pub struct FormatParams {

/// VP9 profile id.
pub profile_id: Option<u32>,

/// AV1 profile.
///
/// Indicates the highest AV1 profile that may have been used to generate
/// the bitstream or that the receiver supports. The range of possible values
/// is identical to the seq_profile syntax element specified in AV1. If the
/// parameter is not present, it MUST be inferred to be 0 (“Main” profile).
///
/// 0 8-bit or 10-bit 4:2:0
/// 1 8-bit or 10-bit 4:4:4
/// 2 8-bit or 10-bit 4:2:2
/// 2 12-bit 4:2:0, 4:2:2, 4:4:4
pub profile: Option<u8>,

/// AV1 level-idx.
///
/// Indicates the highest AV1 level that may have been used to generate the
/// bitstream or that the receiver supports. The range of possible values
/// is identical to the seq_level_idx syntax element specified in AV1. If
/// the parameter is not present, it MUST be inferred to be 5 (level 3.1).
pub level_idx: Option<u8>,

/// AV1 tier.
///
/// Indicates the highest tier that may have been used to generate the bitstream
/// or that the receiver supports. The range of possible values is identical
/// to the seq_tier syntax element specified in AV1. If the parameter is not
/// present, the tier MUST be inferred to be 0.
pub tier: Option<u8>,
}

impl PayloadParams {
Expand Down Expand Up @@ -298,6 +327,10 @@ impl PayloadParams {
return Self::match_vp9_score(c0, c1);
}

if c0.codec == Codec::Av1 {
return Self::match_av1_score(c0, c1);
}

// TODO: Fuzzy matching for any other audio codecs
// TODO: Fuzzy matching for video

Expand Down Expand Up @@ -345,6 +378,37 @@ impl PayloadParams {
Some(100)
}

fn match_av1_score(c0: CodecSpec, c1: CodecSpec) -> Option<usize> {
// TODO: consider media direction for a proper less or equal matching
// The AV1 stream sent by either the offerer or the answerer MUST be
// encoded with a profile, level and tier, lesser or equal to the values
// of the level-idx, profile and tier declared in the SDP by the receiving
// agent.
// https://aomediacodec.github.io/av1-rtp-spec/#723-usage-with-the-sdp-offeranswer-model

// Default values: profile = 0, level-idx = 5, tier = 0
// https://aomediacodec.github.io/av1-rtp-spec/#72-sdp-parameters
let c0_profile = c0.format.profile.unwrap_or(0);
let c1_profile = c1.format.profile.unwrap_or(0);
if c0_profile != c1_profile {
return None;
}

let c0_level_idx = c0.format.level_idx.unwrap_or(5);
let c1_level_idx = c1.format.level_idx.unwrap_or(5);
if c0_level_idx != c1_level_idx {
return None;
}

let c0_tier = c0.format.tier.unwrap_or(0);
let c1_tier = c1.format.tier.unwrap_or(0);
if c0_tier != c1_tier {
return None;
}

Some(100)
}

fn match_h264_score(c0: CodecSpec, c1: CodecSpec) -> Option<usize> {
// Default packetization mode is 0. https://www.rfc-editor.org/rfc/rfc6184#section-6.2
let c0_packetization_mode = c0.format.packetization_mode.unwrap_or(0);
Expand Down Expand Up @@ -812,6 +876,9 @@ impl FormatParams {
PacketizationMode(v) => self.packetization_mode = Some(*v),
ProfileLevelId(v) => self.profile_level_id = Some(*v),
ProfileId(v) => self.profile_id = Some(*v),
Profile(v) => self.profile = Some(*v),
LevelIdx(v) => self.level_idx = Some(*v),
Tier(v) => self.tier = Some(*v),
Apt(_) => {}
Unknown => {}
}
Expand Down Expand Up @@ -842,6 +909,15 @@ impl FormatParams {
if let Some(v) = self.profile_id {
r.push(ProfileId(v));
}
if let Some(v) = self.profile {
r.push(Profile(v));
}
if let Some(v) = self.level_idx {
r.push(LevelIdx(v));
}
if let Some(v) = self.tier {
r.push(Tier(v));
}

r
}
Expand Down Expand Up @@ -944,13 +1020,10 @@ mod test {
clock_rate: Frequency::NINETY_KHZ,
channels: None,
format: FormatParams {
min_p_time: None,
use_inband_fec: None,
use_dtx: None,
level_asymmetry_allowed,
packetization_mode,
profile_level_id,
profile_id: None, // VP8
..Default::default()
},
}
}
Expand Down
180 changes: 114 additions & 66 deletions src/sdp/data.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -918,6 +918,15 @@ pub enum FormatParam {
/// VP9 profile id
ProfileId(u32),

/// AV1 profile
Profile(u8),

/// AV1 level-idx
LevelIdx(u8),

/// AV1 tier
Tier(u8),

/// RTX (resend) codecs, which PT it concerns.
Apt(Pt),

Expand All @@ -929,51 +938,26 @@ impl FormatParam {
pub fn parse(k: &str, v: &str) -> Self {
use FormatParam::*;
match k {
"minptime" => {
if let Ok(v) = v.parse() {
MinPTime(v)
} else {
trace!("Failed to parse: {}", k);
Unknown
}
}
"useinbandfec" => UseInbandFec(v == "1"),
"usedtx" => UseDtx(v == "1"),
"level-asymmetry-allowed" => LevelAsymmetryAllowed(v == "1"),
"packetization-mode" => {
if let Ok(v) = v.parse() {
PacketizationMode(v)
} else {
trace!("Failed to parse: {}", k);
Unknown
}
}
"profile-level-id" => {
if let Ok(v) = u32::from_str_radix(v, 16).or_else(|_| v.parse()) {
ProfileLevelId(v)
} else {
trace!("Failed to parse: {}", k);
Unknown
}
}
"profile-id" => {
if let Ok(v) = v.parse() {
ProfileId(v)
} else {
trace!("Failed to parse: {}", k);
Unknown
}
}
"apt" => {
if let Ok(v) = v.parse::<u8>() {
Apt(v.into())
} else {
trace!("Failed to parse: {}", k);
Unknown
}
}
_ => Unknown,
"minptime" => v.parse().map(MinPTime).ok(),
"useinbandfec" => Some(UseInbandFec(v == "1")),
"usedtx" => Some(UseDtx(v == "1")),
"level-asymmetry-allowed" => Some(LevelAsymmetryAllowed(v == "1")),
"packetization-mode" => v.parse().map(PacketizationMode).ok(),
"profile-level-id" => u32::from_str_radix(v, 16)
.or_else(|_| v.parse())
.map(ProfileLevelId)
.ok(),
"profile-id" => v.parse().map(ProfileId).ok(),
"profile" => v.parse().map(Profile).ok(),
"level-idx" => v.parse().map(LevelIdx).ok(),
"tier" => v.parse().map(Tier).ok(),
"apt" => v.parse::<u8>().map(|v| Apt(Pt::from(v))).ok(),
_ => None,
}
.unwrap_or_else(|| {
trace!("Failed to parse FormatParam: {k}={v}");
Unknown
})
}
}

Expand All @@ -990,6 +974,9 @@ impl fmt::Display for FormatParam {
PacketizationMode(v) => write!(f, "packetization-mode={}", *v),
ProfileLevelId(v) => write!(f, "profile-level-id={:06x}", *v),
ProfileId(v) => write!(f, "profile-id={}", *v),
Profile(v) => write!(f, "profile={}", v),
LevelIdx(v) => write!(f, "level-idx={}", *v),
Tier(v) => write!(f, "tier={}", v),
Apt(v) => write!(f, "apt={v}"),
Unknown => Ok(()),
}
Expand Down Expand Up @@ -1483,27 +1470,14 @@ mod test {
),
],
},
media_lines: vec![MediaLine {
typ: MediaType::Audio,
disabled: false,
proto: Proto::Srtp,
pts: vec![
111.into(),
103.into(),
104.into(),
9.into(),
0.into(),
8.into(),
106.into(),
105.into(),
13.into(),
110.into(),
112.into(),
113.into(),
126.into(),
],
bw: None,
attrs: vec![
media_lines: vec![
MediaLine {
typ: MediaType::Audio,
disabled: false,
proto: Proto::Srtp,
pts: vec![111, 103, 104, 9, 0, 8, 106, 105, 13, 110, 112, 113, 126].into_iter().map(Pt::from).collect(),
bw: None,
attrs: vec![
MediaAttribute::Rtcp("9 IN IP4 0.0.0.0".into()),
MediaAttribute::IceUfrag("S5hk".into()),
MediaAttribute::IcePwd("0zV/Yu3y8aDzbHgqWhnVQhqP".into()),
Expand All @@ -1527,7 +1501,48 @@ mod test {
MediaAttribute::Ssrc { ssrc: 3_948_621_874.into(), attr: "msid".into(), value: "5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK f78dde68-7055-4e20-bb37-433803dd1ed1".into() },
MediaAttribute::Ssrc { ssrc: 3_948_621_874.into(), attr: "mslabel".into(), value: "5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK".into() },
MediaAttribute::Ssrc { ssrc: 3_948_621_874.into(), attr: "label".into(), value: "f78dde68-7055-4e20-bb37-433803dd1ed1".into() }],
}],
},
MediaLine {
typ: MediaType::Video,
disabled: false,
proto: Proto::Srtp,
pts: vec![45.into(), 46.into()],
bw: None,
attrs: vec![
MediaAttribute::Rtcp("9 IN IP4 0.0.0.0".into()),
MediaAttribute::IceUfrag("S5hk".into()),
MediaAttribute::IcePwd("0zV/Yu3y8aDzbHgqWhnVQhqP".into()),
MediaAttribute::IceOptions("trickle".into()),
MediaAttribute::Fingerprint(Fingerprint { hash_func: "sha-256".into(), bytes: vec![140, 100, 237, 3, 118, 208, 61, 180, 136, 8, 145, 100, 8, 128, 168, 198, 90, 191, 139, 78, 56, 39, 150, 202, 8, 73, 37, 115, 70, 96, 32, 220] }),
MediaAttribute::Setup(Setup::ActPass),
MediaAttribute::Mid("1".into()),
MediaAttribute::ExtMap{ id: 14, ext: Extension::TransmissionTimeOffset },
MediaAttribute::ExtMap{ id: 2, ext: Extension::AbsoluteSendTime },
MediaAttribute::ExtMap{ id: 13, ext: Extension::VideoOrientation },
MediaAttribute::ExtMap{ id: 3, ext: Extension::TransportSequenceNumber },
MediaAttribute::ExtMap{ id: 5, ext: Extension::PlayoutDelay },
MediaAttribute::ExtMap{ id: 6, ext: Extension::VideoContentType },
MediaAttribute::ExtMap{ id: 7, ext: Extension::VideoTiming },
MediaAttribute::ExtMap{ id: 8, ext: Extension::ColorSpace },
MediaAttribute::ExtMap{ id: 4, ext: Extension::RtpMid },
MediaAttribute::ExtMap{ id: 10, ext: Extension::RtpStreamId },
MediaAttribute::ExtMap{ id: 11, ext: Extension::RepairedRtpStreamId },
MediaAttribute::SendRecv,
MediaAttribute::Msid(Msid { stream_id: "-".into(), track_id: "4018fd65-ac50-4861-89a4-1f2cc35bbb5e".into() }),
MediaAttribute::RtcpMux,
MediaAttribute::RtcpRsize,
MediaAttribute::RtpMap { pt: 45.into(), value: RtpMap { codec: Codec::Av1, clock_rate: Frequency::NINETY_KHZ, channels: None }},
MediaAttribute::RtcpFb { pt: 45.into(), value: "goog-remb".into() },
MediaAttribute::RtcpFb { pt: 45.into(), value: "transport-cc".into() },
MediaAttribute::RtcpFb { pt: 45.into(), value: "ccm fir".into() },
MediaAttribute::RtcpFb { pt: 45.into(), value: "nack".into() },
MediaAttribute::RtcpFb { pt: 45.into(), value: "nack pli".into() },
MediaAttribute::Fmtp { pt: 45.into(), values: vec![FormatParam::LevelIdx(5), FormatParam::Profile(0), FormatParam::Tier(0)] },
MediaAttribute::RtpMap { pt: 46.into(), value: RtpMap { codec: Codec::Rtx, clock_rate: Frequency::NINETY_KHZ, channels: None } },
MediaAttribute::Fmtp { pt: 46.into(), values: vec![FormatParam::Apt(45.into())] }
],
}
],
};
assert_eq!(&format!("{sdp}"), &format!("v=0\r\n\
o=str0m-{VERSION} 5058682828002148772 2 IN IP4 0.0.0.0\r\n\
Expand Down Expand Up @@ -1560,6 +1575,39 @@ mod test {
a=ssrc:3948621874 msid:5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK f78dde68-7055-4e20-bb37-433803dd1ed1\r\n\
a=ssrc:3948621874 mslabel:5UUdwiuY7OML2EkQtF38pJtNP5v7In1LhjEK\r\n\
a=ssrc:3948621874 label:f78dde68-7055-4e20-bb37-433803dd1ed1\r\n\
m=video 9 UDP/TLS/RTP/SAVPF 45 46\r\n\
c=IN IP4 0.0.0.0\r\n\
a=rtcp:9 IN IP4 0.0.0.0\r\n\
a=ice-ufrag:S5hk\r\n\
a=ice-pwd:0zV/Yu3y8aDzbHgqWhnVQhqP\r\n\
a=ice-options:trickle\r\n\
a=fingerprint:sha-256 8C:64:ED:03:76:D0:3D:B4:88:08:91:64:08:80:A8:C6:5A:BF:8B:4E:38:27:96:CA:08:49:25:73:46:60:20:DC\r\n\
a=setup:actpass\r\n\
a=mid:1\r\n\
a=extmap:14 urn:ietf:params:rtp-hdrext:toffset\r\n\
a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n\
a=extmap:13 urn:3gpp:video-orientation\r\n\
a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n\
a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay\r\n\
a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type\r\n\
a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing\r\n\
a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space\r\n\
a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n\
a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n\
a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n\
a=sendrecv\r\n\
a=msid:- 4018fd65-ac50-4861-89a4-1f2cc35bbb5e\r\n\
a=rtcp-mux\r\n\
a=rtcp-rsize\r\n\
a=rtpmap:45 AV1/90000\r\n\
a=rtcp-fb:45 goog-remb\r\n\
a=rtcp-fb:45 transport-cc\r\n\
a=rtcp-fb:45 ccm fir\r\n\
a=rtcp-fb:45 nack\r\n\
a=rtcp-fb:45 nack pli\r\n\
a=fmtp:45 level-idx=5;profile=0;tier=0\r\n\
a=rtpmap:46 rtx/90000\r\n\
a=fmtp:46 apt=45\r\n\
"));
}
}
Loading
Loading