Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion quest/m2/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,6 @@ ranked just below deterministic TS legs, which it requires.
- [Dart on iOS](/quest/m2/dart-ios.md) - prove the shipped iOS native asset actually loads on a device, which no CI can
- [Mobile completion](/quest/m2/mobile-completion.md) - verify the Swift and Kotlin bindings over Rust-owned capture before closing #700
- [Catalog colour](/quest/m2/color-catalog.md) - the catalog describes a rendition's colour and HDR properties, which the WebGPU HDR renderer reads
- [Display aspect](/quest/m2/display-aspect.md) - H.264 and H.265 (Annex-B and hvcC) importers report a non-square sample aspect as the catalog display aspect, and AV1 too if it carries one
- [WebGPU HDR](/quest/m2/webgpu-hdr.md) - HDR renditions play as HDR where the browser and display can show it, and tone-map to SDR elsewhere
- [Grant stats label](/quest/m2/grant-label.md) - an auth grant's optional `label` names what its session's presence and live bytes count under in stats, apart from the path scope it may access
- [RTSP import](/quest/m2/rtsp-import.md) - `moq import rtsp` publishes an IP camera from its own network, through a `moq-rtsp` crate whose one-session ingest a caller can supervise itself
Expand Down
44 changes: 0 additions & 44 deletions quest/m2/display-aspect.md

This file was deleted.

2 changes: 2 additions & 0 deletions rs/moq-ffi/src/media.rs
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,9 @@ pub struct MoqVideo {
pub track: Option<String>,
pub codec: String,
pub description: Option<Vec<u8>>,
/// The encoded pixel dimensions.
pub coded: Option<MoqDimensions>,
/// The aspect ratio the whole picture is displayed at; absent means the decoded shape.
pub display_aspect: Option<MoqDimensions>,
pub bitrate: Option<u64>,
/// Whether this rendition may be selected; when false, no frames are coming.
Expand Down
4 changes: 4 additions & 0 deletions rs/moq-mux/src/codec/av1/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@
//! Maps the AV1CodecConfigurationRecord (av1C) flag bits into the
//! catalog's AV1 codec struct, and provides an [`Import`] that publishes
//! raw AV1 bitstreams (OBU-framed) to a moq broadcast.
//!
//! The sequence header has no sample aspect. AV1's per-frame
//! [`render_size`](https://aomediacodec.github.io/av1-spec/#render-size-syntax)
//! hint can change between frames, so it does not establish a display aspect for the whole rendition.

mod import;
mod split;
Expand Down
38 changes: 36 additions & 2 deletions rs/moq-mux/src/codec/h264/import.rs
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,9 @@ fn config_from_avcc(avcc_bytes: &[u8]) -> Result<hang::catalog::VideoConfig> {
});
config.coded_width = avcc.coded_width;
config.coded_height = avcc.coded_height;
config.framerate = avcc.sps.first().and_then(|sps| super::sps_framerate(sps));
if let Some(vui) = avcc.sps.first().and_then(|sps| super::sps_vui(sps)) {
vui.apply(&mut config);
}
config.description = Some(Bytes::copy_from_slice(avcc_bytes));
Ok(config)
}
Expand All @@ -266,7 +268,9 @@ fn config_from_sps(sps_nal: &[u8]) -> Result<hang::catalog::VideoConfig> {
});
config.coded_width = Some(sps.coded_width);
config.coded_height = Some(sps.coded_height);
config.framerate = super::sps_framerate(sps_nal);
if let Some(vui) = super::sps_vui(sps_nal) {
vui.apply(&mut config);
}
Ok(config)
}

Expand Down Expand Up @@ -302,6 +306,36 @@ mod tests {
(track, catalog)
}

#[test]
fn annexb_resolves_display_aspect() {
use crate::codec::h264::fixtures;
for (sps, expected) in [
(fixtures::SPS_IPB, (None, None)),
(fixtures::SPS_ANAMORPHIC, (Some(16), Some(9))),
(fixtures::SPS_EXTENDED_SAR, (Some(28), Some(15))),
] {
let mut inline = vec![0, 0, 0, 1];
inline.extend_from_slice(sps);
let config = config(&inline).unwrap();
assert_eq!((config.display_aspect_width, config.display_aspect_height), expected);
}
}

#[test]
fn avcc_resolves_display_aspect() {
use crate::codec::h264::fixtures;
for (sps, expected) in [
(fixtures::SPS_IPB, (None, None)),
(fixtures::SPS_ANAMORPHIC, (Some(16), Some(9))),
(fixtures::SPS_EXTENDED_SAR, (Some(28), Some(15))),
] {
let avcc =
super::super::build_avcc(&[Bytes::from_static(sps)], &[Bytes::from_static(fixtures::PPS)]).unwrap();
let config = config(&avcc).unwrap();
assert_eq!((config.display_aspect_width, config.display_aspect_height), expected);
}
}

/// Only a fixed-rate SPS states the frame rate; without `fixed_frame_rate_flag` its tick is a
/// ceiling, left for the catalog estimator to measure.
#[test]
Expand Down
70 changes: 48 additions & 22 deletions rs/moq-mux/src/codec/h264/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -161,22 +161,10 @@ pub(crate) fn sps_hrd(nal: &[u8]) -> Option<crate::codec::video::Hrd> {
sps_vui(nal)?.hrd
}

/// The frame rate an SPS NAL unit fixes with `fixed_frame_rate_flag`.
///
/// Without the flag the VUI tick only bounds the rate from above, so it is left to measurement.
/// A field-capable SPS is left out too: `pic_struct` in the slice header then decides whether a
/// tick counts a field or a frame (ITU-T H.264 E.2.1, Table E-6), and only measurement settles it.
pub(crate) fn sps_framerate(nal: &[u8]) -> Option<f64> {
let vui = sps_vui(nal)?;
if !vui.frame_only {
return None;
}
let (units, scale) = vui.period?;
Some(scale as f64 / units as f64)
}

/// What an SPS VUI declares about picture timing.
/// What an SPS VUI declares about picture timing and sample shape.
struct Vui {
/// The sample aspect, when the VUI specifies one.
sample_aspect: Option<(u32, u32)>,
/// `frame_mbs_only_flag`: every picture is a frame, so two ticks are one frame.
frame_only: bool,
/// One frame's duration as `(units, scale)` when `fixed_frame_rate_flag` is set.
Expand All @@ -187,6 +175,28 @@ struct Vui {
hrd: Option<crate::codec::video::Hrd>,
}

impl Vui {
/// Only a fixed frame rate on a frame-only SPS fixes the picture period; field pictures
/// need their slice header's `pic_struct` (ITU-T H.264 E.2.1, Table E-6).
fn framerate(&self) -> Option<f64> {
if !self.frame_only {
return None;
}
let (units, scale) = self.period?;
Some(scale as f64 / units as f64)
}

fn apply(&self, config: &mut hang::catalog::VideoConfig) {
config.framerate = self.framerate();
let display_aspect = match (config.coded_width, config.coded_height, self.sample_aspect) {
(Some(width), Some(height), Some(sample)) => crate::codec::video::display_aspect(width, height, sample),
_ => None,
};
config.display_aspect_width = display_aspect.map(|(width, _)| width);
config.display_aspect_height = display_aspect.map(|(_, height)| height);
}
}

/// `None` when the SPS carries no VUI or fails to parse.
fn sps_vui(nal: &[u8]) -> Option<Vui> {
if nal.len() < 4 {
Expand Down Expand Up @@ -274,9 +284,13 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result<Option<Vui>> {
}

// vui_parameters()
if r.read_flag()? && r.read_u8()? == 255 {
r.skip_bits(32)?; // sar_width, sar_height
}
let sample_aspect = match r.read_flag()? {
true => match r.read_u8()? {
255 => Some((u32::from(r.read_u16()?), u32::from(r.read_u16()?))),
idc => crate::codec::video::sample_aspect(idc),
},
false => None,
};
if r.read_flag()? {
r.skip_bits(1)?; // overscan_appropriate_flag
}
Expand Down Expand Up @@ -314,6 +328,7 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result<Option<Vui>> {
r.skip_bits(1)?; // pic_struct_present_flag
if !r.read_flag()? {
return Ok(Some(Vui {
sample_aspect,
frame_only,
period,
depth: None,
Expand All @@ -326,6 +341,7 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result<Option<Vui>> {
}
let depth = read_ue(&mut r)?; // max_num_reorder_frames
Ok(Some(Vui {
sample_aspect,
frame_only,
period,
depth: Some(depth),
Expand Down Expand Up @@ -724,6 +740,16 @@ fn process_nal(
/// tests that need a VUI with a bitstream restriction.
#[cfg(test)]
pub(crate) mod fixtures {
/// A 1440x1080 x264 encode with `setsar=4/3` and `force-cfr=1`, shown 16:9.
pub(crate) const SPS_ANAMORPHIC: &[u8] = &[
0x67, 0x64, 0x00, 0x28, 0xac, 0xd9, 0x40, 0x5a, 0x02, 0x27, 0xe5, 0xc3, 0x84, 0x00, 0x00, 0x03, 0x00, 0x04,
0x00, 0x00, 0x03, 0x00, 0xca, 0x3c, 0x60, 0xc6, 0x58,
];
/// A 64x48 x264 encode with extended 7:5 SAR, shown 28:15.
pub(crate) const SPS_EXTENDED_SAR: &[u8] = &[
0x67, 0x64, 0x00, 0x0a, 0xac, 0xd9, 0x44, 0x7b, 0xff, 0x00, 0x07, 0x00, 0x05, 0x10, 0x00, 0x00, 0x03, 0x00,
0x10, 0x00, 0x00, 0x03, 0x03, 0x28, 0xf1, 0x22, 0x59, 0x60,
];
/// `bframes=1:force-cfr=1`: one B-frame per reference (`max_num_reorder_frames` 1) with
/// `fixed_frame_rate_flag` set.
pub(crate) const SPS_IPB: &[u8] = &[
Expand Down Expand Up @@ -802,12 +828,12 @@ mod tests {

#[test]
fn sps_framerate_needs_a_fixed_rate() {
assert_eq!(sps_framerate(fixtures::SPS_IPB), Some(25.0));
assert_eq!(sps_vui(fixtures::SPS_IPB).unwrap().framerate(), Some(25.0));
// The same 50 Hz tick without `fixed_frame_rate_flag` is only a ceiling.
assert_eq!(sps_framerate(fixtures::SPS_IPB_VARIABLE), None);
assert_eq!(sps_vui(fixtures::SPS_IPB_VARIABLE).unwrap().framerate(), None);
// A field-capable SPS makes the tick a field or a frame depending on `pic_struct`.
assert_eq!(sps_framerate(fixtures::SPS_FIELD), None);
assert_eq!(sps_framerate(&[0x67, 0x42, 0xc0, 0x1f, 0xde]), None);
assert_eq!(sps_vui(fixtures::SPS_FIELD).unwrap().framerate(), None);
assert!(sps_vui(&[0x67, 0x42, 0xc0, 0x1f, 0xde]).is_none());
}

#[test]
Expand Down
Loading
Loading