From 0d70b8c0317ccfb68d065de97166ea05257a2b44 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 16:37:02 +0000 Subject: [PATCH] fix(moq-mux): send the H.265 picture aspect, not the pixel aspect (backport #5574) The H.265 importer copied the VUI sample aspect into the catalog's displayAspectWidth/Height, so a 1280x720 square-pixel stream played square in @moq/watch. It now omits the fields for square samples and sends the whole picture's aspect otherwise, and refuses an SPS whose conformance window crops the picture to nothing. The draft-lcurley-moq-hang.md wording change stays on main, where the draft is versioned. Co-authored-by: Adidharma Torutama Co-Authored-By: Claude Opus 5.5 (cherry picked from commit d91b042d3f4509b0e1382639deba2ccfa159ed82) --- js/hang/src/catalog/video.ts | 5 +-- rs/hang/src/catalog/video/mod.rs | 7 ++-- rs/moq-ffi/src/media.rs | 2 +- rs/moq-mux/src/catalog/tracks.rs | 4 +- rs/moq-mux/src/codec/h265/import.rs | 65 +++++++++++++++++++++++++---- rs/moq-mux/src/codec/h265/mod.rs | 17 ++++++++ rs/moq-mux/src/codec/h265/sps.rs | 44 ++++++++++++++++++- 7 files changed, 126 insertions(+), 18 deletions(-) diff --git a/js/hang/src/catalog/video.ts b/js/hang/src/catalog/video.ts index 53c9946a7fa..a4c23bda8c7 100644 --- a/js/hang/src/catalog/video.ts +++ b/js/hang/src/catalog/video.ts @@ -35,9 +35,8 @@ export const VideoConfigSchema = z.object({ codedWidth: z.optional(u53Schema), codedHeight: z.optional(u53Schema), - // Ratio of display width/height to coded width/height - // Allows stretching/squishing individual "pixels" of the video - // If not provided, the display ratio is 1:1 + // The aspect ratio the whole picture is displayed at, as WebCodecs' VideoDecoderConfig + // displayAspectWidth/displayAspectHeight. Absent means the decoded picture's own shape. displayAspectWidth: z.optional(u53Schema), displayAspectHeight: z.optional(u53Schema), diff --git a/rs/hang/src/catalog/video/mod.rs b/rs/hang/src/catalog/video/mod.rs index f4bba68565b..d299f19fb0e 100644 --- a/rs/hang/src/catalog/video/mod.rs +++ b/rs/hang/src/catalog/video/mod.rs @@ -192,10 +192,11 @@ pub struct VideoConfig { pub coded_width: Option, pub coded_height: Option, - /// The display aspect ratio of the media. + /// The aspect ratio the whole picture is displayed at, as WebCodecs' + /// `VideoDecoderConfig.displayAspectWidth` / `displayAspectHeight`. /// - /// This allows you to stretch/shrink pixels of the video. - /// If not provided, the display aspect ratio is 1:1 + /// Absent means the decoded picture's own shape. A publisher with non-square + /// samples sets the picture's aspect here, never the sample aspect. /// /// The `displayRatio*` aliases decode catalogs from publishers predating the /// rename to `displayAspect*`; the current name is what we emit. diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 5042c51083f..3ecbecf5fd7 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -166,7 +166,7 @@ pub struct MoqVideoHint { /// The encoded pixel dimensions. #[uniffi(default = None)] pub coded: Option, - /// The display aspect ratio. + /// The aspect ratio the whole picture is displayed at, not the sample aspect. Absent means the decoded shape. #[uniffi(default = None)] pub display_aspect: Option, /// The maximum bitrate in bits per second. diff --git a/rs/moq-mux/src/catalog/tracks.rs b/rs/moq-mux/src/catalog/tracks.rs index 80e29e2fb92..1b7bb6c87fc 100644 --- a/rs/moq-mux/src/catalog/tracks.rs +++ b/rs/moq-mux/src/catalog/tracks.rs @@ -190,9 +190,9 @@ pub struct VideoHint { pub coded_width: Option, /// The encoded height in pixels. pub coded_height: Option, - /// The display aspect ratio width. + /// The width of the aspect ratio the whole picture is displayed at, not the sample aspect. pub display_aspect_width: Option, - /// The display aspect ratio height. + /// The height of the aspect ratio the whole picture is displayed at. pub display_aspect_height: Option, /// The maximum bitrate in bits per second. pub bitrate: Option, diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index 0cc85badebe..d97df683771 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -285,8 +285,9 @@ fn config_from_sps(sps_nal: &[u8]) -> Result { .as_ref() .and_then(super::sps_period) .map(|(units, scale)| scale as f64 / units as f64); - config.display_aspect_width = vui_data.display_ratio_width; - config.display_aspect_height = vui_data.display_ratio_height; + let display_aspect = vui_data.display_aspect(head.width, head.height); + config.display_aspect_width = display_aspect.map(|(width, _)| width); + config.display_aspect_height = display_aspect.map(|(_, height)| height); Ok(config) } @@ -309,13 +310,13 @@ fn find_sps(payload: &[u8]) -> Option { #[derive(Default)] struct VuiData { - display_ratio_width: Option, - display_ratio_height: Option, + sample_aspect_width: Option, + sample_aspect_height: Option, } impl VuiData { fn new(vui: &scuffle_h265::VuiParameters) -> Self { - let (display_ratio_width, display_ratio_height) = match &vui.aspect_ratio_info { + let (sample_aspect_width, sample_aspect_height) = match &vui.aspect_ratio_info { // Extended SAR has explicit arbitrary values for width and height. scuffle_h265::AspectRatioInfo::ExtendedSar { sar_width, sar_height } => { (Some(*sar_width as u32), Some(*sar_height as u32)) @@ -327,10 +328,34 @@ impl VuiData { }; VuiData { - display_ratio_width, - display_ratio_height, + sample_aspect_width, + sample_aspect_height, } } + + /// The aspect a `width` by `height` picture is shown at, or none when its samples are square. + fn display_aspect(&self, width: u32, height: u32) -> Option<(u32, u32)> { + let (Some(sample_width), Some(sample_height)) = (self.sample_aspect_width, self.sample_aspect_height) else { + return None; + }; + if sample_width == 0 || sample_height == 0 || sample_width == sample_height { + return None; + } + let shown_width = u64::from(width) * u64::from(sample_width); + let shown_height = u64::from(height) * u64::from(sample_height); + let divisor = gcd(shown_width, shown_height); + Some(( + u32::try_from(shown_width / divisor).ok()?, + u32::try_from(shown_height / divisor).ok()?, + )) + } +} + +fn gcd(mut a: u64, mut b: u64) -> u64 { + while b != 0 { + (a, b) = (b, a % b); + } + a } fn aspect_ratio_from_idc(idc: scuffle_h265::AspectRatioIdc) -> Option<(u32, u32)> { @@ -386,6 +411,32 @@ mod tests { assert_eq!(config(&ceiling).unwrap().framerate, None); } + /// Square samples carry no display aspect, so the picture is shown at its own size. + #[test] + fn square_samples_carry_no_display_aspect() { + let mut nvenc = vec![0, 0, 0, 1]; + nvenc.extend_from_slice(fixtures::SPS_NVENC); + let config = config(&nvenc).unwrap(); + assert_eq!((config.coded_width, config.coded_height), (Some(1280), Some(720))); + assert_eq!( + (config.display_aspect_width, config.display_aspect_height), + (None, None) + ); + } + + /// Non-square samples resolve to the whole picture's aspect: 1440x1080 at 4:3 is shown 16:9. + #[test] + fn anamorphic_samples_resolve_to_the_picture_aspect() { + let mut anamorphic = vec![0, 0, 0, 1]; + anamorphic.extend_from_slice(fixtures::SPS_ANAMORPHIC); + let config = config(&anamorphic).unwrap(); + assert_eq!((config.coded_width, config.coded_height), (Some(1440), Some(1080))); + assert_eq!( + (config.display_aspect_width, config.display_aspect_height), + (Some(16), Some(9)) + ); + } + /// An hvcC initializer resolves a config with the hvcC stored as `description`. /// The hvc1 analogue of the avcC test in [`crate::codec::h264`]'s importer. #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index 9b696577297..bdcce1c0c1b 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -525,6 +525,23 @@ pub(crate) mod fixtures { 0x00, 0x20, 0x05, 0x26, 0x5c, 0x00, 0x33, 0x7f, 0x98, 0x01, ]; + /// SPS from a 1280x720, 60 fps hevc_nvenc encode (`p1`, `ull`), the desktop host's shape. Its VUI + /// sets `aspect_ratio_idc` 1: square samples. + pub(crate) const SPS_NVENC: &[u8] = &[ + 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, + 0xa0, 0x02, 0x80, 0x80, 0x2e, 0x1f, 0x13, 0x96, 0x5d, 0x29, 0x08, 0x46, 0x45, 0xd5, 0x0c, 0x04, 0x04, 0x00, + 0x00, 0x03, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0xf1, 0x80, 0x17, 0x79, 0x44, 0x00, 0x02, 0xdc, 0x6c, 0x00, + 0x00, 0x2d, 0xc6, 0xc0, 0x40, + ]; + + /// SPS from a 1440x1080 hevc_nvenc encode with 4:3 samples (`setsar=4/3`), shown 16:9. + pub(crate) const SPS_ANAMORPHIC: &[u8] = &[ + 0x42, 0x01, 0x01, 0x01, 0x60, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x78, + 0xa0, 0x02, 0xd0, 0x80, 0x11, 0x07, 0xcb, 0x96, 0x5d, 0x29, 0x08, 0x46, 0x45, 0xd5, 0x0c, 0x38, 0x04, 0x00, + 0x00, 0x03, 0x00, 0x04, 0x00, 0x00, 0x03, 0x00, 0x79, 0x80, 0x17, 0x79, 0x44, 0x00, 0x02, 0xdc, 0x6c, 0x00, + 0x00, 0xf4, 0x24, 0x04, + ]; + /// An hvcC record built from the real parameter sets (4-byte NALU lengths). pub(crate) fn hvcc() -> Bytes { super::build_hvcc( diff --git a/rs/moq-mux/src/codec/h265/sps.rs b/rs/moq-mux/src/codec/h265/sps.rs index 9e2c15e47bd..f1fbaecc2d2 100644 --- a/rs/moq-mux/src/codec/h265/sps.rs +++ b/rs/moq-mux/src/codec/h265/sps.rs @@ -60,6 +60,27 @@ impl Sps { impl SpsHead { fn from_rbsp(rbsp: &scuffle_h265::SpsRbsp) -> Result { let profile = &rbsp.profile_tier_level.general_profile; + // scuffle_h265's `cropped_width` subtracts unchecked, so crop here and refuse a conformance + // window that leaves no picture, as `read` does. + let window = &rbsp.conformance_window; + let crop = |size: u64, sub: u8, start: u64, end: u64| { + let offset = start.checked_add(end)?.checked_mul(sub.into())?; + u32::try_from(size.checked_sub(offset)?).ok().filter(|size| *size > 0) + }; + let width = crop( + rbsp.pic_width_in_luma_samples.get(), + rbsp.sub_width_c(), + window.conf_win_left_offset, + window.conf_win_right_offset, + ) + .ok_or(Error::SpsParse)?; + let height = crop( + rbsp.pic_height_in_luma_samples.get(), + rbsp.sub_height_c(), + window.conf_win_top_offset, + window.conf_win_bottom_offset, + ) + .ok_or(Error::SpsParse)?; Ok(Self { profile_space: profile.profile_space, tier_flag: profile.tier_flag, @@ -72,8 +93,8 @@ impl SpsHead { chroma_format_idc: rbsp.chroma_format_idc, bit_depth_luma_minus8: rbsp.bit_depth_luma_minus8, bit_depth_chroma_minus8: rbsp.bit_depth_chroma_minus8, - width: rbsp.cropped_width() as u32, - height: rbsp.cropped_height() as u32, + width, + height, }) } @@ -241,6 +262,25 @@ mod tests { ); } + /// A conformance window that crops away the whole picture, or past it, is refused rather than + /// handing a zero (or wrapped) size to the display aspect. + #[test] + fn empty_conformance_window_is_refused() { + let rbsp = SpsNALUnit::parse(&mut &fixtures::SPS[..]).unwrap().rbsp; + let width = rbsp.pic_width_in_luma_samples.get() / u64::from(rbsp.sub_width_c()); + for right in [width, width + 1] { + let mut rbsp = rbsp.clone(); + rbsp.conformance_window.conf_win_left_offset = 0; + rbsp.conformance_window.conf_win_right_offset = right; + assert!(SpsHead::from_rbsp(&rbsp).is_err()); + } + let height = rbsp.pic_height_in_luma_samples.get() / u64::from(rbsp.sub_height_c()); + let mut rbsp = rbsp.clone(); + rbsp.conformance_window.conf_win_top_offset = 0; + rbsp.conformance_window.conf_win_bottom_offset = height; + assert!(SpsHead::from_rbsp(&rbsp).is_err()); + } + /// The fallback covers a refusal after the head, not a truncated or invalid head. #[test] fn truncated_or_invalid_head_is_still_an_error() {