From 23ee04ba8cb20d2bd3f1dfc30d7acc6293627ebb Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:09:00 -0700 Subject: [PATCH 1/3] chore: claim display aspect importer quest From 3ae586150ddeffb79517c7cdedf6a9171ea52e46 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 11:47:15 -0700 Subject: [PATCH 2/3] fix(mux): resolve display aspect from shared SPS metadata Co-Authored-By: GPT-6 --- quest/m2/display-aspect.md | 49 +++---- rs/moq-ffi/src/media.rs | 2 + rs/moq-mux/src/codec/av1/mod.rs | 4 + rs/moq-mux/src/codec/h264/import.rs | 38 +++++- rs/moq-mux/src/codec/h264/mod.rs | 70 ++++++---- rs/moq-mux/src/codec/h265/import.rs | 127 +++---------------- rs/moq-mux/src/codec/h265/mod.rs | 44 +++++-- rs/moq-mux/src/codec/video.rs | 41 ++++++ rs/moq-mux/src/container/fmp4/import.rs | 16 +++ rs/moq-mux/src/container/fmp4/import_test.rs | 124 +++++++++++++++++- rs/moq-mux/src/container/mkv/import.rs | 27 ++-- rs/moq-mux/src/container/mkv/import_test.rs | 53 ++++++++ 12 files changed, 414 insertions(+), 181 deletions(-) diff --git a/quest/m2/display-aspect.md b/quest/m2/display-aspect.md index c7bbbfd8d2b..6a0d01629bf 100644 --- a/quest/m2/display-aspect.md +++ b/quest/m2/display-aspect.md @@ -14,30 +14,31 @@ today. ## Plan -[#5574](https://github.com/moq-dev/moq/pull/5574) fixed the H.265 Annex-B path -(`config_from_sps`, `rs/moq-mux/src/codec/h265/import.rs`). The remaining -gaps: - -- `config_from_hvcc` sets only the coded size and description. Share the - Annex-B path's VUI read rather than duplicating it. -- The H.264 SPS parser skips `sar_width`/`sar_height`. -- AV1 has no sample aspect in its sequence header. Check whether - `render_size` should count as a display aspect before adding it. If not, - record why and leave AV1 out. -- `MoqVideo.display_aspect` (and `coded`) in `rs/moq-ffi/src/media.rs` have - no doc line. - -Keep #5574's choice that an encode hint fills in only square samples. - -Tests: a non-square fixture per codec path, and a square one that stays -unset. - -Not planned: an upstream scuffle_h265 issue for the unchecked arithmetic in -`cropped_width()`/`cropped_height()`. We never call them, since our SPS -reader crops with checked math. - -Public API: doc only. Wire: the catalog's existing display aspect fields are -populated in more cases. +The known-SPS paths now share their metadata: H.264 reads predefined and +extended VUI sample aspect through Annex-B and avcC; HEVC uses the same SPS/VUI +resolver for Annex-B and hvcC. Matroska and fMP4's avc1, avc3, hvc1 and hev1 +initializers also take that metadata. Square samples leave the display aspect +unset, so the existing hint remains a fallback. The FFI's existing dimensions +and aspect fields are documented. + +AV1's sequence header carries no sample aspect. Its +[render size](https://aomediacodec.github.io/av1-spec/#render-size-syntax) is a +per-frame hint that may inherit reference-frame dimensions, so it does not +establish a fixed aspect for the rendition. + +A scope decision remains: valid avc3/hev1 initializers may omit parameter sets +and carry the SPS only inside later CMAF media. The passthrough importer does +not inspect those payloads, so their bitstream aspect stays unknown. Decide +whether that acquisition belongs here or in a follow-up before closing this +quest. The current fix preserves headerless initialization and does not add +media-payload inspection. + +Tests cover non-square and square SPS fixtures through the codec, Matroska and +all four fMP4 paths, plus headerless avc3/hev1 controls. The metadata regressions +fail before their fixes. + +Public API: no shape change; existing FFI fields gain docs. Wire: existing +catalog aspect and fixed-frame-rate fields are populated in more cases. ## Related diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 76c6f425b8b..3e45deaae87 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -105,7 +105,9 @@ pub struct MoqVideo { pub track: Option, pub codec: String, pub description: Option>, + /// The encoded pixel dimensions. pub coded: Option, + /// The aspect ratio the whole picture is displayed at; absent means the decoded shape. pub display_aspect: Option, pub bitrate: Option, /// Whether this rendition may be selected; when false, no frames are coming. diff --git a/rs/moq-mux/src/codec/av1/mod.rs b/rs/moq-mux/src/codec/av1/mod.rs index c29d98985ed..8affd3a9abc 100644 --- a/rs/moq-mux/src/codec/av1/mod.rs +++ b/rs/moq-mux/src/codec/av1/mod.rs @@ -3,6 +3,10 @@ //! Maps the AV1CodecConfigurationRecord (av1C) flag bits into the //! catalog's AV1 codec struct, and provides an [`Import`] that publishes //! raw AV1 bitstreams (OBU-framed) to a moq broadcast. +//! +//! The sequence header has no sample aspect. AV1's per-frame +//! [`render_size`](https://aomediacodec.github.io/av1-spec/#render-size-syntax) +//! hint can change between frames, so it does not establish a display aspect for the whole rendition. mod import; mod split; diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index b2fe8a31649..d90984184f5 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -248,7 +248,9 @@ fn config_from_avcc(avcc_bytes: &[u8]) -> Result { }); config.coded_width = avcc.coded_width; config.coded_height = avcc.coded_height; - config.framerate = avcc.sps.first().and_then(|sps| super::sps_framerate(sps)); + if let Some(vui) = avcc.sps.first().and_then(|sps| super::sps_vui(sps)) { + vui.apply(&mut config); + } config.description = Some(Bytes::copy_from_slice(avcc_bytes)); Ok(config) } @@ -266,7 +268,9 @@ fn config_from_sps(sps_nal: &[u8]) -> Result { }); config.coded_width = Some(sps.coded_width); config.coded_height = Some(sps.coded_height); - config.framerate = super::sps_framerate(sps_nal); + if let Some(vui) = super::sps_vui(sps_nal) { + vui.apply(&mut config); + } Ok(config) } @@ -302,6 +306,36 @@ mod tests { (track, catalog) } + #[test] + fn annexb_resolves_display_aspect() { + use crate::codec::h264::fixtures; + for (sps, expected) in [ + (fixtures::SPS_IPB, (None, None)), + (fixtures::SPS_ANAMORPHIC, (Some(16), Some(9))), + (fixtures::SPS_EXTENDED_SAR, (Some(28), Some(15))), + ] { + let mut inline = vec![0, 0, 0, 1]; + inline.extend_from_slice(sps); + let config = config(&inline).unwrap(); + assert_eq!((config.display_aspect_width, config.display_aspect_height), expected); + } + } + + #[test] + fn avcc_resolves_display_aspect() { + use crate::codec::h264::fixtures; + for (sps, expected) in [ + (fixtures::SPS_IPB, (None, None)), + (fixtures::SPS_ANAMORPHIC, (Some(16), Some(9))), + (fixtures::SPS_EXTENDED_SAR, (Some(28), Some(15))), + ] { + let avcc = + super::super::build_avcc(&[Bytes::from_static(sps)], &[Bytes::from_static(fixtures::PPS)]).unwrap(); + let config = config(&avcc).unwrap(); + assert_eq!((config.display_aspect_width, config.display_aspect_height), expected); + } + } + /// Only a fixed-rate SPS states the frame rate; without `fixed_frame_rate_flag` its tick is a /// ceiling, left for the catalog estimator to measure. #[test] diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index 634a2c2c21e..471a00eab09 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -161,22 +161,10 @@ pub(crate) fn sps_hrd(nal: &[u8]) -> Option { sps_vui(nal)?.hrd } -/// The frame rate an SPS NAL unit fixes with `fixed_frame_rate_flag`. -/// -/// Without the flag the VUI tick only bounds the rate from above, so it is left to measurement. -/// A field-capable SPS is left out too: `pic_struct` in the slice header then decides whether a -/// tick counts a field or a frame (ITU-T H.264 E.2.1, Table E-6), and only measurement settles it. -pub(crate) fn sps_framerate(nal: &[u8]) -> Option { - let vui = sps_vui(nal)?; - if !vui.frame_only { - return None; - } - let (units, scale) = vui.period?; - Some(scale as f64 / units as f64) -} - -/// What an SPS VUI declares about picture timing. +/// What an SPS VUI declares about picture timing and sample shape. struct Vui { + /// The sample aspect, when the VUI specifies one. + sample_aspect: Option<(u32, u32)>, /// `frame_mbs_only_flag`: every picture is a frame, so two ticks are one frame. frame_only: bool, /// One frame's duration as `(units, scale)` when `fixed_frame_rate_flag` is set. @@ -187,6 +175,28 @@ struct Vui { hrd: Option, } +impl Vui { + /// Only a fixed frame rate on a frame-only SPS fixes the picture period; field pictures + /// need their slice header's `pic_struct` (ITU-T H.264 E.2.1, Table E-6). + fn framerate(&self) -> Option { + if !self.frame_only { + return None; + } + let (units, scale) = self.period?; + Some(scale as f64 / units as f64) + } + + fn apply(&self, config: &mut hang::catalog::VideoConfig) { + config.framerate = self.framerate(); + let display_aspect = match (config.coded_width, config.coded_height, self.sample_aspect) { + (Some(width), Some(height), Some(sample)) => crate::codec::video::display_aspect(width, height, sample), + _ => None, + }; + config.display_aspect_width = display_aspect.map(|(width, _)| width); + config.display_aspect_height = display_aspect.map(|(_, height)| height); + } +} + /// `None` when the SPS carries no VUI or fails to parse. fn sps_vui(nal: &[u8]) -> Option { if nal.len() < 4 { @@ -274,9 +284,13 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result> { } // vui_parameters() - if r.read_flag()? && r.read_u8()? == 255 { - r.skip_bits(32)?; // sar_width, sar_height - } + let sample_aspect = match r.read_flag()? { + true => match r.read_u8()? { + 255 => Some((u32::from(r.read_u16()?), u32::from(r.read_u16()?))), + idc => crate::codec::video::sample_aspect(idc), + }, + false => None, + }; if r.read_flag()? { r.skip_bits(1)?; // overscan_appropriate_flag } @@ -314,6 +328,7 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result> { r.skip_bits(1)?; // pic_struct_present_flag if !r.read_flag()? { return Ok(Some(Vui { + sample_aspect, frame_only, period, depth: None, @@ -326,6 +341,7 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result> { } let depth = read_ue(&mut r)?; // max_num_reorder_frames Ok(Some(Vui { + sample_aspect, frame_only, period, depth: Some(depth), @@ -724,6 +740,16 @@ fn process_nal( /// tests that need a VUI with a bitstream restriction. #[cfg(test)] pub(crate) mod fixtures { + /// A 1440x1080 x264 encode with `setsar=4/3` and `force-cfr=1`, shown 16:9. + pub(crate) const SPS_ANAMORPHIC: &[u8] = &[ + 0x67, 0x64, 0x00, 0x28, 0xac, 0xd9, 0x40, 0x5a, 0x02, 0x27, 0xe5, 0xc3, 0x84, 0x00, 0x00, 0x03, 0x00, 0x04, + 0x00, 0x00, 0x03, 0x00, 0xca, 0x3c, 0x60, 0xc6, 0x58, + ]; + /// A 64x48 x264 encode with extended 7:5 SAR, shown 28:15. + pub(crate) const SPS_EXTENDED_SAR: &[u8] = &[ + 0x67, 0x64, 0x00, 0x0a, 0xac, 0xd9, 0x44, 0x7b, 0xff, 0x00, 0x07, 0x00, 0x05, 0x10, 0x00, 0x00, 0x03, 0x00, + 0x10, 0x00, 0x00, 0x03, 0x03, 0x28, 0xf1, 0x22, 0x59, 0x60, + ]; /// `bframes=1:force-cfr=1`: one B-frame per reference (`max_num_reorder_frames` 1) with /// `fixed_frame_rate_flag` set. pub(crate) const SPS_IPB: &[u8] = &[ @@ -802,12 +828,12 @@ mod tests { #[test] fn sps_framerate_needs_a_fixed_rate() { - assert_eq!(sps_framerate(fixtures::SPS_IPB), Some(25.0)); + assert_eq!(sps_vui(fixtures::SPS_IPB).unwrap().framerate(), Some(25.0)); // The same 50 Hz tick without `fixed_frame_rate_flag` is only a ceiling. - assert_eq!(sps_framerate(fixtures::SPS_IPB_VARIABLE), None); + assert_eq!(sps_vui(fixtures::SPS_IPB_VARIABLE).unwrap().framerate(), None); // A field-capable SPS makes the tick a field or a frame depending on `pic_struct`. - assert_eq!(sps_framerate(fixtures::SPS_FIELD), None); - assert_eq!(sps_framerate(&[0x67, 0x42, 0xc0, 0x1f, 0xde]), None); + assert_eq!(sps_vui(fixtures::SPS_FIELD).unwrap().framerate(), None); + assert!(sps_vui(&[0x67, 0x42, 0xc0, 0x1f, 0xde]).is_none()); } #[test] diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index 63aac840cc4..e8574c5d0ef 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -175,7 +175,7 @@ impl Import { if self.last_sps.as_ref() == Some(sps_nal) { return Ok(()); } - let config = config_from_sps(sps_nal)?; + let config = super::config_from_sps(sps_nal, true)?; self.last_sps = Some(sps_nal.clone()); self.apply_config(config)?; Ok(()) @@ -231,39 +231,7 @@ pub fn config(buf: &[u8]) -> Result { return Ok(super::config_from_hvcc(buf)?); } let sps = find_sps(buf).ok_or(Error::NotInitialized)?; - config_from_sps(&sps) -} - -/// The config an inline SPS describes (`in_band: true`, the hev1 shape). No `description`: the -/// parameter sets ride with every keyframe. -fn config_from_sps(sps_nal: &[u8]) -> Result { - let super::Sps { head, full } = super::Sps::parse(sps_nal)?; - // The VUI (frame rate, aspect ratio) only comes with scuffle_h265's full parse. - let vui_data = full - .as_ref() - .and_then(|sps| sps.vui_parameters.as_ref()) - .map(VuiData::new) - .unwrap_or_default(); - - let mut config = hang::catalog::VideoConfig::new(hang::catalog::H265 { - in_band: true, // An inline SPS is the hev1 shape; hvc1 configs come from `config_from_hvcc`. - profile_space: head.profile_space, - profile_idc: head.profile_idc, - profile_compatibility_flags: head.profile_compatibility_flags, - tier_flag: head.tier_flag, - level_idc: head.level_idc, - constraint_flags: head.constraint_flags, - }); - config.coded_width = Some(head.width); - config.coded_height = Some(head.height); - config.framerate = full - .as_ref() - .and_then(super::sps_period) - .map(|(units, scale)| scale as f64 / units as f64); - let display_aspect = vui_data.display_aspect(head.width, head.height); - config.display_aspect_width = display_aspect.map(|(width, _)| width); - config.display_aspect_height = display_aspect.map(|(_, height)| height); - Ok(config) + Ok(super::config_from_sps(&sps, true)?) } fn is_sps(nal: &[u8]) -> bool { @@ -283,80 +251,6 @@ fn find_sps(payload: &[u8]) -> Option { nals.flush().ok().flatten().filter(|nal| is_sps(nal)) } -#[derive(Default)] -struct VuiData { - sample_aspect_width: Option, - sample_aspect_height: Option, -} - -impl VuiData { - fn new(vui: &scuffle_h265::VuiParameters) -> Self { - let (sample_aspect_width, sample_aspect_height) = match &vui.aspect_ratio_info { - // Extended SAR has explicit arbitrary values for width and height. - scuffle_h265::AspectRatioInfo::ExtendedSar { sar_width, sar_height } => { - (Some(*sar_width as u32), Some(*sar_height as u32)) - } - // Predefined map to known values. - scuffle_h265::AspectRatioInfo::Predefined(idc) => aspect_ratio_from_idc(*idc) - .map(|(w, h)| (Some(w), Some(h))) - .unwrap_or((None, None)), - }; - - VuiData { - sample_aspect_width, - sample_aspect_height, - } - } - - /// The aspect a `width` by `height` picture is shown at, or none when its samples are square. - fn display_aspect(&self, width: u32, height: u32) -> Option<(u32, u32)> { - let (Some(sample_width), Some(sample_height)) = (self.sample_aspect_width, self.sample_aspect_height) else { - return None; - }; - if sample_width == 0 || sample_height == 0 || sample_width == sample_height { - return None; - } - let shown_width = u64::from(width) * u64::from(sample_width); - let shown_height = u64::from(height) * u64::from(sample_height); - let divisor = gcd(shown_width, shown_height); - Some(( - u32::try_from(shown_width / divisor).ok()?, - u32::try_from(shown_height / divisor).ok()?, - )) - } -} - -fn gcd(mut a: u64, mut b: u64) -> u64 { - while b != 0 { - (a, b) = (b, a % b); - } - a -} - -fn aspect_ratio_from_idc(idc: scuffle_h265::AspectRatioIdc) -> Option<(u32, u32)> { - match idc { - scuffle_h265::AspectRatioIdc::Unspecified => None, - scuffle_h265::AspectRatioIdc::Square => Some((1, 1)), - scuffle_h265::AspectRatioIdc::Aspect12_11 => Some((12, 11)), - scuffle_h265::AspectRatioIdc::Aspect10_11 => Some((10, 11)), - scuffle_h265::AspectRatioIdc::Aspect16_11 => Some((16, 11)), - scuffle_h265::AspectRatioIdc::Aspect40_33 => Some((40, 33)), - scuffle_h265::AspectRatioIdc::Aspect24_11 => Some((24, 11)), - scuffle_h265::AspectRatioIdc::Aspect20_11 => Some((20, 11)), - scuffle_h265::AspectRatioIdc::Aspect32_11 => Some((32, 11)), - scuffle_h265::AspectRatioIdc::Aspect80_33 => Some((80, 33)), - scuffle_h265::AspectRatioIdc::Aspect18_11 => Some((18, 11)), - scuffle_h265::AspectRatioIdc::Aspect15_11 => Some((15, 11)), - scuffle_h265::AspectRatioIdc::Aspect64_33 => Some((64, 33)), - scuffle_h265::AspectRatioIdc::Aspect160_99 => Some((160, 99)), - scuffle_h265::AspectRatioIdc::Aspect4_3 => Some((4, 3)), - scuffle_h265::AspectRatioIdc::Aspect3_2 => Some((3, 2)), - scuffle_h265::AspectRatioIdc::Aspect2_1 => Some((2, 1)), - scuffle_h265::AspectRatioIdc::ExtendedSar => None, - _ => None, // Reserved - } -} - #[cfg(test)] mod tests { use bytes::BytesMut; @@ -412,6 +306,23 @@ mod tests { ); } + #[test] + fn hvcc_resolves_display_aspect() { + for (sps, expected) in [ + (fixtures::SPS_NVENC, (None, None)), + (fixtures::SPS_ANAMORPHIC, (Some(16), Some(9))), + ] { + let hvcc = super::super::build_hvcc( + &[Bytes::from_static(fixtures::VPS)], + &[Bytes::from_static(sps)], + &[Bytes::from_static(fixtures::PPS)], + ) + .unwrap(); + let config = config(&hvcc).unwrap(); + assert_eq!((config.display_aspect_width, config.display_aspect_height), expected); + } + } + /// An hvcC initializer resolves a config with the hvcC stored as `description`. /// The hvc1 analogue of the avcC test in [`crate::codec::h264`]'s importer. #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index b9061aa2bdd..41b7178b26c 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -199,20 +199,42 @@ impl Hvcc { pub(crate) fn config_from_hvcc(hvcc: &[u8]) -> Result { let params = Hvcc::parse(hvcc)?; let sps_nal = params.sps.first().ok_or(Error::MissingSps)?; - let sps = Sps::parse(sps_nal)?.head; + let mut config = config_from_sps(sps_nal, false)?; + config.description = Some(Bytes::copy_from_slice(hvcc)); + Ok(config) +} +/// Resolve either sample framing from the same SPS dimensions and VUI metadata. +pub(crate) fn config_from_sps(sps_nal: &[u8], in_band: bool) -> Result { + let Sps { head, full } = Sps::parse(sps_nal)?; let mut config = hang::catalog::VideoConfig::new(hang::catalog::H265 { - in_band: false, - profile_space: sps.profile_space, - profile_idc: sps.profile_idc, - profile_compatibility_flags: sps.profile_compatibility_flags, - tier_flag: sps.tier_flag, - level_idc: sps.level_idc, - constraint_flags: sps.constraint_flags, + in_band, + profile_space: head.profile_space, + profile_idc: head.profile_idc, + profile_compatibility_flags: head.profile_compatibility_flags, + tier_flag: head.tier_flag, + level_idc: head.level_idc, + constraint_flags: head.constraint_flags, }); - config.coded_width = Some(sps.width); - config.coded_height = Some(sps.height); - config.description = Some(Bytes::copy_from_slice(hvcc)); + config.coded_width = Some(head.width); + config.coded_height = Some(head.height); + config.framerate = full + .as_ref() + .and_then(sps_period) + .map(|(units, scale)| scale as f64 / units as f64); + let sample_aspect = full + .as_ref() + .and_then(|sps| sps.vui_parameters.as_ref()) + .and_then(|vui| match vui.aspect_ratio_info { + scuffle_h265::AspectRatioInfo::ExtendedSar { sar_width, sar_height } => { + Some((u32::from(sar_width), u32::from(sar_height))) + } + scuffle_h265::AspectRatioInfo::Predefined(idc) => crate::codec::video::sample_aspect(idc.0), + }); + let display_aspect = + sample_aspect.and_then(|sample| crate::codec::video::display_aspect(head.width, head.height, sample)); + config.display_aspect_width = display_aspect.map(|(width, _)| width); + config.display_aspect_height = display_aspect.map(|(_, height)| height); Ok(config) } diff --git a/rs/moq-mux/src/codec/video.rs b/rs/moq-mux/src/codec/video.rs index 7a5a64fd672..35a58ce1484 100644 --- a/rs/moq-mux/src/codec/video.rs +++ b/rs/moq-mux/src/codec/video.rs @@ -8,6 +8,47 @@ use crate::catalog::VideoHint; +/// The predefined sample aspects shared by H.264 and H.265 (their VUI Table E-1). +pub(super) fn sample_aspect(idc: u8) -> Option<(u32, u32)> { + Some(match idc { + 1 => (1, 1), + 2 => (12, 11), + 3 => (10, 11), + 4 => (16, 11), + 5 => (40, 33), + 6 => (24, 11), + 7 => (20, 11), + 8 => (32, 11), + 9 => (80, 33), + 10 => (18, 11), + 11 => (15, 11), + 12 => (64, 33), + 13 => (160, 99), + 14 => (4, 3), + 15 => (3, 2), + 16 => (2, 1), + _ => return None, + }) +} + +/// Resolve non-square samples to the whole picture's reduced display aspect. +pub(super) fn display_aspect(width: u32, height: u32, sample: (u32, u32)) -> Option<(u32, u32)> { + let (sample_width, sample_height) = sample; + if width == 0 || height == 0 || sample_width == 0 || sample_height == 0 || sample_width == sample_height { + return None; + } + let shown_width = u64::from(width) * u64::from(sample_width); + let shown_height = u64::from(height) * u64::from(sample_height); + let (mut divisor, mut remainder) = (shown_width, shown_height); + while remainder != 0 { + (divisor, remainder) = (remainder, divisor % remainder); + } + Some(( + u32::try_from(shown_width / divisor).ok()?, + u32::try_from(shown_height / divisor).ok()?, + )) +} + type Track = crate::container::Producer; /// The reordering a video sequence header declares. diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index 6f63d9383ea..4659e627245 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -510,6 +510,7 @@ impl Import { ) -> Result { let mut description = BytesMut::new(); avcc.encode_body(&mut description)?; + let metadata = crate::codec::h264::config(&description)?; let mut config = VideoConfig::new(H264 { profile: avcc.avc_profile_indication, @@ -519,6 +520,9 @@ impl Import { }); config.coded_width = Some(visual.width as _); config.coded_height = Some(visual.height as _); + config.framerate = metadata.framerate; + config.display_aspect_width = metadata.display_aspect_width; + config.display_aspect_height = metadata.display_aspect_height; config.description = Some(description.freeze()); config.container = container; Ok(config) @@ -546,6 +550,18 @@ impl Import { config.description = Some(description.freeze()); config.coded_width = Some(visual.width as _); config.coded_height = Some(visual.height as _); + if let Some(sps) = hvcc + .arrays + .iter() + .filter(|array| array.nal_unit_type == scuffle_h265::NALUnitType::SpsNut.0) + .flat_map(|array| &array.nalus) + .next() + { + let metadata = crate::codec::h265::config_from_sps(sps, in_band)?; + config.framerate = metadata.framerate; + config.display_aspect_width = metadata.display_aspect_width; + config.display_aspect_height = metadata.display_aspect_height; + } config.container = container; Ok(config) } diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 6abecc99a9d..10992ff5baa 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -1,6 +1,6 @@ use futures::FutureExt; use hang::catalog::Container; -use mp4_atom::{Decode, Encode}; +use mp4_atom::{Atom, Decode, Encode}; /// A drift budget no test timeline comes close to, so every group is read. /// @@ -66,6 +66,128 @@ fn decode_init(init: &[u8]) -> (mp4_atom::Ftyp, mp4_atom::Moov) { (ftyp, moov) } +fn video_config(codec: mp4_atom::Codec) -> hang::catalog::VideoConfig { + let (ftyp, mut moov) = decode_init(include_bytes!("test_data/bbb.mp4")); + let trak = moov + .trak + .iter_mut() + .find(|trak| matches!(trak.mdia.minf.stbl.stsd.codecs.first(), Some(mp4_atom::Codec::Avc1(_)))) + .unwrap(); + trak.mdia.minf.stbl.stsd.codecs = vec![codec]; + let mut init = Vec::new(); + ftyp.encode(&mut init).unwrap(); + moov.encode(&mut init).unwrap(); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + import.decode(&init).unwrap(); + import.finish().unwrap(); + catalog.snapshot().video.renditions.values().next().unwrap().clone() +} + +#[test] +fn sample_entries_resolve_display_aspect() { + use crate::codec::{h264, h265}; + for (avc, hevc, expected) in [ + (h264::fixtures::SPS_IPB, h265::fixtures::SPS_NVENC, (None, None)), + ( + h264::fixtures::SPS_ANAMORPHIC, + h265::fixtures::SPS_ANAMORPHIC, + (Some(16), Some(9)), + ), + ] { + let avcc = mp4_atom::Avcc::new(avc, h264::fixtures::PPS).unwrap(); + let mut bytes = h265::build_hvcc( + &[bytes::Bytes::from_static(h265::fixtures::VPS)], + &[bytes::Bytes::from_static(hevc)], + &[bytes::Bytes::from_static(h265::fixtures::PPS)], + ) + .unwrap(); + let hvcc = mp4_atom::Hvcc::decode_body(&mut bytes).unwrap(); + let mut avc_visual = mp4_atom::Visual { + width: 1440, + height: 1080, + ..Default::default() + }; + let mut hevc_visual = avc_visual.clone(); + if expected == (None, None) { + avc_visual.width = 64; + avc_visual.height = 64; + hevc_visual.width = 1280; + hevc_visual.height = 720; + } + for codec in [ + mp4_atom::Codec::Avc1(mp4_atom::Avc1 { + avcc: avcc.clone(), + visual: avc_visual.clone(), + ..Default::default() + }), + mp4_atom::Codec::Avc3(mp4_atom::Avc3 { + avcc: avcc.clone(), + visual: avc_visual.clone(), + ..Default::default() + }), + mp4_atom::Codec::Hvc1(mp4_atom::Hvc1 { + hvcc: hvcc.clone(), + visual: hevc_visual.clone(), + ..Default::default() + }), + mp4_atom::Codec::Hev1(mp4_atom::Hev1 { + hvcc: hvcc.clone(), + visual: hevc_visual.clone(), + ..Default::default() + }), + ] { + let config = video_config(codec); + assert_eq!( + (config.display_aspect_width, config.display_aspect_height), + expected, + "{}", + config.codec + ); + } + } +} + +#[test] +fn in_band_entries_without_parameter_sets_keep_visual_dimensions() { + let mut avcc = + mp4_atom::Avcc::new(crate::codec::h264::fixtures::SPS_IPB, crate::codec::h264::fixtures::PPS).unwrap(); + avcc.sequence_parameter_sets.clear(); + avcc.picture_parameter_sets.clear(); + let mut bytes = crate::codec::h265::fixtures::hvcc(); + let mut hvcc = mp4_atom::Hvcc::decode_body(&mut bytes).unwrap(); + hvcc.arrays.clear(); + let visual = mp4_atom::Visual { + width: 720, + height: 576, + ..Default::default() + }; + for codec in [ + mp4_atom::Codec::Avc3(mp4_atom::Avc3 { + avcc, + visual: visual.clone(), + ..Default::default() + }), + mp4_atom::Codec::Hev1(mp4_atom::Hev1 { + hvcc, + visual, + ..Default::default() + }), + ] { + let config = video_config(codec); + assert_eq!((config.coded_width, config.coded_height), (Some(720), Some(576))); + assert_eq!( + (config.display_aspect_width, config.display_aspect_height), + (None, None) + ); + assert!( + matches!(&config.codec, hang::catalog::VideoCodec::H264(h264) if h264.inline) + || matches!(&config.codec, hang::catalog::VideoCodec::H265(h265) if h265.in_band) + ); + } +} + #[test] fn test_bbb_catalog() { let data = include_bytes!("test_data/bbb.mp4"); diff --git a/rs/moq-mux/src/container/mkv/import.rs b/rs/moq-mux/src/container/mkv/import.rs index 48d8891dd9c..56df560bf7f 100644 --- a/rs/moq-mux/src/container/mkv/import.rs +++ b/rs/moq-mux/src/container/mkv/import.rs @@ -4,7 +4,7 @@ use std::io::Cursor; use crate::Result; use bytes::{Buf, Bytes, BytesMut}; -use hang::catalog::{AAC, AudioCodec, AudioConfig, H264, H265, VP9, VideoCodec, VideoConfig}; +use hang::catalog::{AAC, AudioCodec, AudioConfig, H265, VP9, VideoCodec, VideoConfig}; use moq_net::Timestamp; use mp4_atom::Atom; use webm_iterable::WebmIterator; @@ -813,18 +813,7 @@ fn build_h264_config(codec_private: Option<&Bytes>) -> Result { codec_id: "V_MPEG4/ISO/AVC", purpose: "AVCDecoderConfigurationRecord", })?; - let avcc = crate::codec::h264::Avcc::parse(avcc_bytes)?; - - let mut config = VideoConfig::new(H264 { - profile: avcc.profile, - constraints: avcc.constraints, - level: avcc.level, - inline: false, - }); - config.description = Some(avcc_bytes.clone()); - config.coded_width = avcc.coded_width; - config.coded_height = avcc.coded_height; - Ok(config) + crate::codec::h264::config(avcc_bytes) } fn build_h265_config(codec_private: Option<&Bytes>) -> Result { @@ -848,6 +837,18 @@ fn build_h265_config(codec_private: Option<&Bytes>) -> Result { constraint_flags: hvcc.general_constraint_indicator_flags, }); config.description = Some(description.freeze()); + if let Some(sps) = hvcc + .arrays + .iter() + .filter(|array| array.nal_unit_type == scuffle_h265::NALUnitType::SpsNut.0) + .flat_map(|array| &array.nalus) + .next() + { + let metadata = crate::codec::h265::config_from_sps(sps, false)?; + config.framerate = metadata.framerate; + config.display_aspect_width = metadata.display_aspect_width; + config.display_aspect_height = metadata.display_aspect_height; + } Ok(config) } diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index a5d692df88a..1e8a5da3a80 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -188,6 +188,59 @@ fn run(data: &[u8]) -> crate::catalog::hang::Catalog { catalog.snapshot() } +#[test] +fn codec_private_resolves_display_aspect() { + use crate::codec::{h264, h265}; + use bytes::Bytes; + for (avc, hevc, expected) in [ + (h264::fixtures::SPS_IPB, h265::fixtures::SPS_NVENC, (None, None)), + ( + h264::fixtures::SPS_ANAMORPHIC, + h265::fixtures::SPS_ANAMORPHIC, + (Some(16), Some(9)), + ), + ] { + let avcc = h264::build_avcc(&[Bytes::from_static(avc)], &[Bytes::from_static(h264::fixtures::PPS)]).unwrap(); + let hvcc = h265::build_hvcc( + &[Bytes::from_static(h265::fixtures::VPS)], + &[Bytes::from_static(hevc)], + &[Bytes::from_static(h265::fixtures::PPS)], + ) + .unwrap(); + for (codec_id, private) in [("V_MPEG4/ISO/AVC", avcc), ("V_MPEGH/ISO/HEVC", hvcc)] { + let (width, height) = match expected { + (None, None) if codec_id == "V_MPEG4/ISO/AVC" => (64, 64), + (None, None) => (1280, 720), + _ => (1440, 1080), + }; + let data = MkvBuilder::new() + .header("matroska") + .segment_start() + .info(1_000_000) + .tracks(vec![MatroskaSpec::TrackEntry(Master::Full(vec![ + MatroskaSpec::TrackNumber(1), + MatroskaSpec::TrackUID(1), + MatroskaSpec::TrackType(1), + MatroskaSpec::CodecID(codec_id.into()), + MatroskaSpec::CodecPrivate(private.to_vec()), + MatroskaSpec::Video(Master::Full(vec![ + MatroskaSpec::PixelWidth(width), + MatroskaSpec::PixelHeight(height), + ])), + ]))]) + .segment_end() + .build(); + let catalog = run(&data); + let config = catalog.video.renditions.values().next().unwrap(); + assert_eq!( + (config.display_aspect_width, config.display_aspect_height), + expected, + "{codec_id}" + ); + } + } +} + #[test] fn test_flac_catalog() { let data = MkvBuilder::new() From 6c2cfb86ff50e0d7f733bee1203dc2d828662667 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 21:32:17 -0700 Subject: [PATCH 3/3] refactor(mux): share the hvcC SPS metadata read and complete the display aspect quest The known-SPS fix lands here; headerless avc3/hev1 acquisition and container aspect signals (pasp, Matroska DisplayWidth) are left for follow-ups. Co-Authored-By: Claude Opus 5.5 --- quest/m2/README.md | 1 - quest/m2/display-aspect.md | 45 ------------------------- rs/moq-mux/src/codec/h265/mod.rs | 20 +++++++++++ rs/moq-mux/src/container/fmp4/import.rs | 13 +------ rs/moq-mux/src/container/mkv/import.rs | 13 +------ 5 files changed, 22 insertions(+), 70 deletions(-) delete mode 100644 quest/m2/display-aspect.md diff --git a/quest/m2/README.md b/quest/m2/README.md index 3cf0e15c423..580123ee0b8 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -90,7 +90,6 @@ ranked just below deterministic TS legs, which it requires. - [Mobile completion](/quest/m2/mobile-completion.md) - verify the Swift and Kotlin bindings over Rust-owned capture before closing #700 - [#709](/quest/m2/709-automatic-letsencrypt-support.md) - the relay provisions and renews its own ACME certificate through rustls-acme over TLS-ALPN-01, persisted on disk - [Catalog colour](/quest/m2/color-catalog.md) - the catalog describes a rendition's colour and HDR properties, which the WebGPU HDR renderer reads -- [Display aspect](/quest/m2/display-aspect.md) - H.264 and H.265 (Annex-B and hvcC) importers report a non-square sample aspect as the catalog display aspect, and AV1 too if it carries one - [WebGPU HDR](/quest/m2/webgpu-hdr.md) - HDR renditions play as HDR where the browser and display can show it, and tone-map to SDR elsewhere - [Grant stats label](/quest/m2/grant-label.md) - an auth grant's optional `label` names what its session's presence and live bytes count under in stats, apart from the path scope it may access - [RTSP import](/quest/m2/rtsp-import.md) - `moq import rtsp` publishes an IP camera from its own network, through a `moq-rtsp` crate whose one-session ingest a caller can supervise itself diff --git a/quest/m2/display-aspect.md b/quest/m2/display-aspect.md deleted file mode 100644 index 6a0d01629bf..00000000000 --- a/quest/m2/display-aspect.md +++ /dev/null @@ -1,45 +0,0 @@ -# [S] Every video importer reports the display aspect - -## Goal - -A video rendition with non-square samples gets the catalog display aspect -from its bitstream, whichever importer reads it: - -- H.264 (the VUI sample aspect) -- H.265 from both Annex-B and an `hvcC` record -- AV1, if it carries a stream-level signal - -A square-sample stream still leaves the fields unset, as the encode hint does -today. - -## Plan - -The known-SPS paths now share their metadata: H.264 reads predefined and -extended VUI sample aspect through Annex-B and avcC; HEVC uses the same SPS/VUI -resolver for Annex-B and hvcC. Matroska and fMP4's avc1, avc3, hvc1 and hev1 -initializers also take that metadata. Square samples leave the display aspect -unset, so the existing hint remains a fallback. The FFI's existing dimensions -and aspect fields are documented. - -AV1's sequence header carries no sample aspect. Its -[render size](https://aomediacodec.github.io/av1-spec/#render-size-syntax) is a -per-frame hint that may inherit reference-frame dimensions, so it does not -establish a fixed aspect for the rendition. - -A scope decision remains: valid avc3/hev1 initializers may omit parameter sets -and carry the SPS only inside later CMAF media. The passthrough importer does -not inspect those payloads, so their bitstream aspect stays unknown. Decide -whether that acquisition belongs here or in a follow-up before closing this -quest. The current fix preserves headerless initialization and does not add -media-payload inspection. - -Tests cover non-square and square SPS fixtures through the codec, Matroska and -all four fMP4 paths, plus headerless avc3/hev1 controls. The metadata regressions -fail before their fixes. - -Public API: no shape change; existing FFI fields gain docs. Wire: existing -catalog aspect and fixed-frame-rate fields are populated in more cases. - -## Related - -- [Catalog colour](/quest/m2/color-catalog.md) - parses the same H.264/H.265 VUI for colour diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index 41b7178b26c..d0e8ac4af48 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -204,6 +204,26 @@ pub(crate) fn config_from_hvcc(hvcc: &[u8]) -> Result Result<()> { + let Some(sps) = hvcc + .arrays + .iter() + .filter(|array| array.nal_unit_type == NALUnitType::SpsNut.0) + .flat_map(|array| &array.nalus) + .next() + else { + return Ok(()); + }; + let metadata = config_from_sps(sps, false)?; + config.framerate = metadata.framerate; + config.display_aspect_width = metadata.display_aspect_width; + config.display_aspect_height = metadata.display_aspect_height; + Ok(()) +} + /// Resolve either sample framing from the same SPS dimensions and VUI metadata. pub(crate) fn config_from_sps(sps_nal: &[u8], in_band: bool) -> Result { let Sps { head, full } = Sps::parse(sps_nal)?; diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index e29fb346ee1..c598b614ba0 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -550,18 +550,7 @@ impl Import { config.description = Some(description.freeze()); config.coded_width = Some(visual.width as _); config.coded_height = Some(visual.height as _); - if let Some(sps) = hvcc - .arrays - .iter() - .filter(|array| array.nal_unit_type == scuffle_h265::NALUnitType::SpsNut.0) - .flat_map(|array| &array.nalus) - .next() - { - let metadata = crate::codec::h265::config_from_sps(sps, in_band)?; - config.framerate = metadata.framerate; - config.display_aspect_width = metadata.display_aspect_width; - config.display_aspect_height = metadata.display_aspect_height; - } + crate::codec::h265::apply_hvcc_sps(hvcc, &mut config)?; config.container = container; Ok(config) } diff --git a/rs/moq-mux/src/container/mkv/import.rs b/rs/moq-mux/src/container/mkv/import.rs index 56df560bf7f..c45f6062e77 100644 --- a/rs/moq-mux/src/container/mkv/import.rs +++ b/rs/moq-mux/src/container/mkv/import.rs @@ -837,18 +837,7 @@ fn build_h265_config(codec_private: Option<&Bytes>) -> Result { constraint_flags: hvcc.general_constraint_indicator_flags, }); config.description = Some(description.freeze()); - if let Some(sps) = hvcc - .arrays - .iter() - .filter(|array| array.nal_unit_type == scuffle_h265::NALUnitType::SpsNut.0) - .flat_map(|array| &array.nalus) - .next() - { - let metadata = crate::codec::h265::config_from_sps(sps, false)?; - config.framerate = metadata.framerate; - config.display_aspect_width = metadata.display_aspect_width; - config.display_aspect_height = metadata.display_aspect_height; - } + crate::codec::h265::apply_hvcc_sps(&hvcc, &mut config)?; Ok(config) }