From 62dd181c82edb0d9d54c2b8e51e8d4459540a514 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 20:19:44 -0700 Subject: [PATCH 01/14] quest(m2): claim color-catalog From b3a4e4147bb6f88f4320fbae611073748bcde709 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 20:36:31 -0700 Subject: [PATCH 02/14] feat(hang): catalog colour model, filled from bitstream and container Co-Authored-By: Claude Opus 5.5 --- rs/hang/src/catalog/root.rs | 1 + rs/hang/src/catalog/video/color.rs | 133 ++++++++++ rs/hang/src/catalog/video/mod.rs | 10 +- rs/moq-mux/src/catalog/tracks.rs | 4 + rs/moq-mux/src/codec/av1/import.rs | 50 +++- rs/moq-mux/src/codec/av1/mod.rs | 2 + rs/moq-mux/src/codec/color.rs | 328 ++++++++++++++++++++++++ rs/moq-mux/src/codec/h264/import.rs | 33 ++- rs/moq-mux/src/codec/h264/mod.rs | 24 +- rs/moq-mux/src/codec/h265/import.rs | 37 ++- rs/moq-mux/src/codec/h265/mod.rs | 9 +- rs/moq-mux/src/codec/h265/sps.rs | 19 ++ rs/moq-mux/src/codec/mod.rs | 1 + rs/moq-mux/src/codec/video.rs | 27 +- rs/moq-mux/src/codec/vp9/import.rs | 3 +- rs/moq-mux/src/codec/vp9/mod.rs | 15 +- rs/moq-mux/src/container/flv/import.rs | 1 + rs/moq-mux/src/container/fmp4/import.rs | 42 ++- rs/moq-mux/src/container/fmp4/mod.rs | 45 ++++ rs/moq-mux/src/container/mkv/export.rs | 39 +++ rs/moq-mux/src/container/mkv/import.rs | 77 ++++++ 21 files changed, 849 insertions(+), 51 deletions(-) create mode 100644 rs/hang/src/catalog/video/color.rs create mode 100644 rs/moq-mux/src/codec/color.rs diff --git a/rs/hang/src/catalog/root.rs b/rs/hang/src/catalog/root.rs index d0e20718a82..ebd5009cff8 100644 --- a/rs/hang/src/catalog/root.rs +++ b/rs/hang/src/catalog/root.rs @@ -438,6 +438,7 @@ mod test { coded_height: None, display_aspect_width: None, display_aspect_height: None, + color: None, bitrate: None, enabled: true, preference: 0, diff --git a/rs/hang/src/catalog/video/color.rs b/rs/hang/src/catalog/video/color.rs new file mode 100644 index 00000000000..a539d153612 --- /dev/null +++ b/rs/hang/src/catalog/video/color.rs @@ -0,0 +1,133 @@ +use serde::{Deserialize, Serialize}; + +/// The colour a rendition's pixels carry, so a renderer can set up its pipeline without reparsing the bitstream. +/// +/// The first four fields are ITU-T H.273 (ISO/IEC 23091-2) code points, the numbering H.264 and +/// H.265 VUI, AV1, VP9, and the ISOBMFF `colr` box all share. Absent means the source left it +/// unspecified, never a signalled default. +/// +/// Marked `#[non_exhaustive]`: start from [`Color::default`] and assign the fields you know. +#[serde_with::skip_serializing_none] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Default)] +#[serde(rename_all = "camelCase")] +#[non_exhaustive] +pub struct Color { + /// The chromaticities of the source primaries, as H.273 `ColourPrimaries` (1 is BT.709, 9 is BT.2020). + pub primaries: Option, + + /// The transfer function, as H.273 `TransferCharacteristics` (1 is BT.709, 16 is PQ, 18 is HLG). + pub transfer: Option, + + /// The matrix between RGB and Y'CbCr, as H.273 `MatrixCoefficients` (0 is RGB, 1 is BT.709, 9 is BT.2020 NCL). + pub matrix: Option, + + /// True for full-range samples, false for limited (studio) range. + pub full_range: Option, + + /// The display the content was mastered on, as SMPTE ST 2086. + pub mastering: Option, + + /// The content's light levels, as CTA-861.3. + pub content_light: Option, +} + +impl Color { + /// True when no field is set, so the colour can be omitted. + pub fn is_empty(&self) -> bool { + self == &Self::default() + } +} + +/// The colour volume of a mastering display, as SMPTE ST 2086. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct Mastering { + /// The red primary. + pub red: Chromaticity, + /// The green primary. + pub green: Chromaticity, + /// The blue primary. + pub blue: Chromaticity, + /// The white point. + pub white: Chromaticity, + /// The peak luminance in candelas per square metre. + pub max_luminance: f64, + /// The minimum luminance in candelas per square metre. + pub min_luminance: f64, +} + +/// A CIE 1931 xy chromaticity coordinate. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq)] +pub struct Chromaticity { + pub x: f64, + pub y: f64, +} + +/// Content light levels in candelas per square metre, as CTA-861.3. Zero means unknown. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ContentLight { + /// The brightest pixel in the content (MaxCLL). + pub max_cll: u16, + /// The brightest frame average in the content (MaxFALL). + pub max_fall: u16, +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn hdr10_round_trips() { + let color = Color { + primaries: Some(9), + transfer: Some(16), + matrix: Some(9), + full_range: Some(false), + mastering: Some(Mastering { + red: Chromaticity { x: 0.708, y: 0.292 }, + green: Chromaticity { x: 0.17, y: 0.797 }, + blue: Chromaticity { x: 0.131, y: 0.046 }, + white: Chromaticity { x: 0.3127, y: 0.329 }, + max_luminance: 1000.0, + min_luminance: 0.005, + }), + content_light: Some(ContentLight { + max_cll: 1000, + max_fall: 400, + }), + }; + + let encoded = serde_json::to_value(&color).unwrap(); + assert_eq!( + encoded, + serde_json::json!({ + "primaries": 9, + "transfer": 16, + "matrix": 9, + "fullRange": false, + "mastering": { + "red": { "x": 0.708, "y": 0.292 }, + "green": { "x": 0.17, "y": 0.797 }, + "blue": { "x": 0.131, "y": 0.046 }, + "white": { "x": 0.3127, "y": 0.329 }, + "maxLuminance": 1000.0, + "minLuminance": 0.005, + }, + "contentLight": { "maxCll": 1000, "maxFall": 400 }, + }) + ); + assert_eq!(serde_json::from_value::(encoded).unwrap(), color); + } + + #[test] + fn unspecified_fields_are_omitted() { + let color = Color { + matrix: Some(1), + ..Default::default() + }; + assert_eq!(serde_json::to_value(&color).unwrap(), serde_json::json!({ "matrix": 1 })); + assert!(!color.is_empty()); + assert!(Color::default().is_empty()); + } +} diff --git a/rs/hang/src/catalog/video/mod.rs b/rs/hang/src/catalog/video/mod.rs index 483428bd556..14ac4172d04 100644 --- a/rs/hang/src/catalog/video/mod.rs +++ b/rs/hang/src/catalog/video/mod.rs @@ -1,11 +1,13 @@ mod av1; mod codec; +mod color; mod h264; mod h265; mod vp9; pub use av1::*; pub use codec::*; +pub use color::*; pub use h264::*; pub use h265::*; pub use vp9::*; @@ -229,7 +231,12 @@ pub struct VideoConfig { #[serde(alias = "displayRatioHeight")] pub display_aspect_height: Option, - // TODO color space + /// The colour the decoded pixels carry, so a renderer need not guess or reparse the bitstream. + /// + /// Live state, not identity: a publisher may fill it in once the bitstream reveals it. + #[serde(default)] + pub color: Option, + /// The maximum bitrate of the video track, if known. #[serde(default)] pub bitrate: Option, @@ -318,6 +325,7 @@ impl VideoConfig { coded_height: None, display_aspect_width: None, display_aspect_height: None, + color: None, bitrate: None, enabled: true, preference: 0, diff --git a/rs/moq-mux/src/catalog/tracks.rs b/rs/moq-mux/src/catalog/tracks.rs index 893fa3b2597..70d6a478e18 100644 --- a/rs/moq-mux/src/catalog/tracks.rs +++ b/rs/moq-mux/src/catalog/tracks.rs @@ -194,6 +194,8 @@ pub struct VideoHint { pub display_aspect_width: Option, /// The height of the aspect ratio the whole picture is displayed at. pub display_aspect_height: Option, + /// The colour the pixels carry. Each field the stream signals outranks the hint's. + pub color: Option, /// The maximum bitrate in bits per second. pub bitrate: Option, /// The frame rate in frames per second. @@ -238,6 +240,7 @@ impl From for VideoHint { coded_height: config.coded_height, display_aspect_width: config.display_aspect_width, display_aspect_height: config.display_aspect_height, + color: config.color, bitrate: config.bitrate, framerate: config.framerate, optimize_for_latency: config.optimize_for_latency, @@ -263,6 +266,7 @@ impl VideoHint { fill(&mut config.coded_height, self.coded_height); fill(&mut config.display_aspect_width, self.display_aspect_width); fill(&mut config.display_aspect_height, self.display_aspect_height); + crate::codec::color::fill(&mut config.color, self.color.as_ref(), "hint"); fill(&mut config.bitrate, self.bitrate); fill(&mut config.framerate, self.framerate); fill(&mut config.optimize_for_latency, self.optimize_for_latency); diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index 46ae5a70afa..daae5fab7bd 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -142,6 +142,7 @@ impl Import { }); config.coded_width = Some(seq_header.max_frame_width as u32); config.coded_height = Some(seq_header.max_frame_height as u32); + config.color = sequence_color(seq_header); self.apply_config(config)?; Ok(()) } @@ -239,10 +240,12 @@ impl Import { /// keyframe's inline sequence header and refining the catalog jitter. fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { for frame in frames { - if frame.keyframe - && let Some(seq) = find_sequence_header(&frame.payload) - { - self.configure_from_seq(&seq)?; + if frame.keyframe { + // HDR metadata first, so a keyframe that also configures publishes once. + self.catalog.hdr(&mut self.track, find_hdr(&frame.payload))?; + if let Some(seq) = find_sequence_header(&frame.payload) { + self.configure_from_seq(&seq)?; + } } // A keyframe we couldn't configure (no sequence header) is undecodable. @@ -277,6 +280,23 @@ fn parse_sequence_header(obu: &[u8]) -> Result> { Ok(SequenceHeaderObu::parse(header, &mut reader).ok()) } +/// The colour a sequence header's `color_config` signals; its range is always explicit. +fn sequence_color(seq_header: &SequenceHeaderObu) -> Option { + let color = &seq_header.color_config; + crate::codec::color::cicp( + color.color_primaries, + color.transfer_characteristics, + color.matrix_coefficients, + Some(color.full_color_range), + ) +} + +/// The colour of the first parseable sequence header in a run of OBUs, such as av1C's `configOBUs`. +pub(crate) fn color(obus: &[u8]) -> Option { + let sequence = parse_sequence_header(&find_sequence_header(obus)?).ok()??; + sequence_color(&sequence) +} + /// Read encoded dimensions from the first parseable sequence header in a frame. pub(crate) fn dimensions(payload: &[u8]) -> Result> { let Some(sequence) = find_sequence_header(payload) else { @@ -291,6 +311,28 @@ pub(crate) fn dimensions(payload: &[u8]) -> Result> { ))) } +/// The HDR metadata a temporal unit's metadata OBUs carry. +fn find_hdr(payload: &Bytes) -> crate::codec::color::Hdr { + let mut hdr = crate::codec::color::Hdr::default(); + let mut scan = |obu: &[u8]| { + let mut reader = obu; + if let Ok(header) = ObuHeader::parse(&mut reader) + && header.obu_type == ObuType::Metadata + { + hdr.update(crate::codec::color::av1_metadata(reader)); + } + }; + let mut buf = payload.clone(); + let mut obus = ObuIterator::new(&mut buf); + while let Some(Ok(obu)) = obus.next() { + scan(&obu); + } + if let Ok(Some(obu)) = obus.flush() { + scan(&obu); + } + hdr +} + /// Find the first sequence-header OBU in a payload, if any. fn find_sequence_header(payload: &[u8]) -> Option { let mut buf = Bytes::copy_from_slice(payload); diff --git a/rs/moq-mux/src/codec/av1/mod.rs b/rs/moq-mux/src/codec/av1/mod.rs index c29d98985ed..7629195ce5b 100644 --- a/rs/moq-mux/src/codec/av1/mod.rs +++ b/rs/moq-mux/src/codec/av1/mod.rs @@ -71,6 +71,8 @@ pub(crate) fn config_from_av1c(av1c: &[u8]) -> Result) -> Option { + // 2 is unspecified for all three and 3 is reserved; 0 is reserved for primaries and transfer + // but is the identity (RGB) matrix. Cameras ship zeroed descriptions, so a reserved value is + // treated as unsignalled rather than refusing the stream. + let known = |code: u8, zero: bool| (code != 2 && code != 3 && (zero || code != 0)).then_some(code); + let mut color = Color::default(); + color.primaries = known(primaries, false); + color.transfer = known(transfer, false); + color.matrix = known(matrix, true); + color.full_range = full_range; + (!color.is_empty()).then_some(color) +} + +/// Fill the absent fields of `color` from `fallback`, a lower-precedence source named `source`. +/// +/// A field both set keeps `color`'s value and logs the disagreement, so a conflict between the +/// bitstream and its container is diagnosable rather than invisible. +pub(crate) fn fill(color: &mut Option, fallback: Option<&Color>, source: &str) { + let Some(fallback) = fallback else { + return; + }; + let color = color.get_or_insert_with(Color::default); + fn field(name: &str, source: &str, slot: &mut Option, other: &Option) { + match (&*slot, other) { + (None, _) => *slot = other.clone(), + (Some(kept), Some(other)) if kept != other => { + tracing::warn!(field = name, ?kept, ?other, source, "colour disagrees with the stream; keeping the stream's"); + } + _ => {} + } + } + field("primaries", source, &mut color.primaries, &fallback.primaries); + field("transfer", source, &mut color.transfer, &fallback.transfer); + field("matrix", source, &mut color.matrix, &fallback.matrix); + field("full_range", source, &mut color.full_range, &fallback.full_range); + field("mastering", source, &mut color.mastering, &fallback.mastering); + field("content_light", source, &mut color.content_light, &fallback.content_light); +} + +/// The static HDR metadata an access unit carries in band. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub(crate) struct Hdr { + pub mastering: Option, + pub content_light: Option, +} + +impl Hdr { + /// Keep any field `other` sets, so an access unit carrying only one message keeps the other. + pub(crate) fn update(&mut self, other: Hdr) { + self.mastering = other.mastering.or(self.mastering); + self.content_light = other.content_light.or(self.content_light); + } + + /// Set these fields on `config`'s colour; they come from the bitstream, so they replace a hint. + pub(crate) fn apply(&self, config: &mut hang::catalog::VideoConfig) { + if *self == Self::default() { + return; + } + let color = config.color.get_or_insert_with(Color::default); + color.mastering = self.mastering.or(color.mastering); + color.content_light = self.content_light.or(color.content_light); + } +} + +/// The HDR metadata in an H.264 or H.265 access unit's SEI NAL units, which are Annex-B, or +/// length-prefixed when `length_size` is set. `header` is the codec's NAL header length and `sei` +/// picks the SEI NAL units by their first byte. +pub(crate) fn access_unit( + payload: &bytes::Bytes, + length_size: Option, + header: usize, + sei: impl Fn(u8) -> bool, +) -> Hdr { + let mut hdr = Hdr::default(); + let mut scan = |nal: &[u8]| { + if nal.len() > header && sei(nal[0]) { + hdr.update(self::sei(&h264_parser::nal::ebsp_to_rbsp(&nal[header..]))); + } + }; + match length_size { + Some(size) => { + let Ok(nals) = crate::codec::annexb::length_prefixed_nals(payload, size) else { + return hdr; + }; + nals.map_while(Result::ok).for_each(&mut scan); + } + None => { + let mut buf = payload.clone(); + let mut nals = crate::codec::annexb::NalIterator::new(&mut buf); + while let Some(Ok(nal)) = nals.next() { + scan(&nal); + } + if let Ok(Some(nal)) = nals.flush() { + scan(&nal); + } + } + } + hdr +} + +const SEI_MASTERING_DISPLAY: u32 = 137; +const SEI_CONTENT_LIGHT: u32 = 144; + +/// The HDR messages in an H.264 or H.265 SEI RBSP (the NAL payload after its header, emulation +/// prevention removed). Both codecs share the `sei_message` syntax and these two payloads +/// (ITU-T H.264 D.1.29 and D.1.31, H.265 D.2.28 and D.2.35). A malformed message ends the scan. +pub(crate) fn sei(rbsp: &[u8]) -> Hdr { + let mut hdr = Hdr::default(); + let mut buf = rbsp; + // Each message needs at least a type and size byte; the last byte is the RBSP stop bit. + while buf.len() > 2 { + let Some(kind) = sei_value(&mut buf) else { break }; + let Some(size) = sei_value(&mut buf) else { break }; + let Some((payload, rest)) = buf.split_at_checked(size as usize) else { + break; + }; + buf = rest; + match kind { + SEI_MASTERING_DISPLAY if payload.len() >= 24 => hdr.mastering = Some(sei_mastering(payload)), + SEI_CONTENT_LIGHT if payload.len() >= 4 => { + hdr.content_light = Some(ContentLight { + max_cll: u16::from_be_bytes([payload[0], payload[1]]), + max_fall: u16::from_be_bytes([payload[2], payload[3]]), + }) + } + _ => {} + } + } + hdr +} + +const AV1_METADATA_HDR_CLL: u64 = 1; +const AV1_METADATA_HDR_MDCV: u64 = 2; + +/// The HDR metadata in an AV1 metadata OBU's payload (AV1 5.8.1, 6.7.3 and 6.7.4): content light +/// levels, or a mastering display whose chromaticities are 0.16 fixed point and whose luminances +/// are 24.8 (max) and 18.14 (min) fixed point. Anything else, or a short payload, is ignored. +pub(crate) fn av1_metadata(payload: &[u8]) -> Hdr { + let mut hdr = Hdr::default(); + let mut buf = payload; + let Some(kind) = leb128(&mut buf) else { + return hdr; + }; + let u16_at = |i: usize| f64::from(u16::from_be_bytes([buf[i], buf[i + 1]])); + let u32_at = |i: usize| f64::from(u32::from_be_bytes([buf[i], buf[i + 1], buf[i + 2], buf[i + 3]])); + match kind { + AV1_METADATA_HDR_CLL if buf.len() >= 4 => { + hdr.content_light = Some(ContentLight { + max_cll: u16_at(0) as u16, + max_fall: u16_at(2) as u16, + }) + } + AV1_METADATA_HDR_MDCV if buf.len() >= 24 => { + let xy = |i: usize| Chromaticity { + x: u16_at(i) / 65_536.0, + y: u16_at(i + 2) / 65_536.0, + }; + hdr.mastering = Some(Mastering { + red: xy(0), + green: xy(4), + blue: xy(8), + white: xy(12), + max_luminance: u32_at(16) / 256.0, + min_luminance: u32_at(20) / 16_384.0, + }) + } + _ => {} + } + hdr +} + +/// An AV1 `leb128()` value. +fn leb128(buf: &mut &[u8]) -> Option { + let mut value = 0u64; + for i in 0..8 { + let (&byte, rest) = buf.split_first()?; + *buf = rest; + value |= u64::from(byte & 0x7F) << (7 * i); + if byte & 0x80 == 0 { + return Some(value); + } + } + None +} + +/// An SEI `payloadType` or `payloadSize`: a run of 0xFF bytes, each adding 255, then the remainder. +fn sei_value(buf: &mut &[u8]) -> Option { + let mut value = 0u32; + loop { + let (&byte, rest) = buf.split_first()?; + *buf = rest; + value = value.checked_add(byte.into())?; + if byte != 0xFF { + return Some(value); + } + } +} + +/// A `mastering_display_colour_volume` payload. Chromaticities count 0.00002 and luminance +/// 0.0001 cd/m². The primaries are conventionally green, blue, red, which libavcodec assumes too. +fn sei_mastering(p: &[u8]) -> Mastering { + let u16_at = |i: usize| u16::from_be_bytes([p[i], p[i + 1]]); + let u32_at = |i: usize| u32::from_be_bytes([p[i], p[i + 1], p[i + 2], p[i + 3]]); + let xy = |i: usize| Chromaticity { + x: f64::from(u16_at(i)) / 50_000.0, + y: f64::from(u16_at(i + 2)) / 50_000.0, + }; + Mastering { + green: xy(0), + blue: xy(4), + red: xy(8), + white: xy(12), + max_luminance: f64::from(u32_at(16)) / 10_000.0, + min_luminance: f64::from(u32_at(20)) / 10_000.0, + } +} + +#[cfg(test)] +pub(crate) mod test { + use super::*; + + /// The HDR10 mastering display and light levels the fixtures below encode. + pub(crate) fn hdr10() -> Hdr { + Hdr { + mastering: Some(Mastering { + red: Chromaticity { x: 0.708, y: 0.292 }, + green: Chromaticity { x: 0.17, y: 0.797 }, + blue: Chromaticity { x: 0.131, y: 0.046 }, + white: Chromaticity { x: 0.3127, y: 0.329 }, + max_luminance: 1000.0, + min_luminance: 0.005, + }), + content_light: Some(ContentLight { + max_cll: 1000, + max_fall: 400, + }), + } + } + + /// An SEI RBSP carrying [`hdr10`]: a mastering display then a content light level message. + pub(crate) fn hdr10_sei() -> Vec { + let mut rbsp = vec![137, 24]; + for v in [8500u16, 39850, 6550, 2300, 35400, 14600, 15635, 16450] { + rbsp.extend_from_slice(&v.to_be_bytes()); + } + rbsp.extend_from_slice(&10_000_000u32.to_be_bytes()); + rbsp.extend_from_slice(&50u32.to_be_bytes()); + rbsp.extend_from_slice(&[144, 4]); + rbsp.extend_from_slice(&1000u16.to_be_bytes()); + rbsp.extend_from_slice(&400u16.to_be_bytes()); + rbsp.push(0x80); + rbsp + } + + #[test] + fn cicp_drops_unspecified_and_reserved() { + assert_eq!(cicp(2, 2, 2, None), None); + assert_eq!(cicp(0, 0, 3, None), None); + let color = cicp(9, 16, 0, Some(false)).unwrap(); + assert_eq!( + (color.primaries, color.transfer, color.matrix, color.full_range), + (Some(9), Some(16), Some(0), Some(false)) + ); + } + + #[test] + fn fill_keeps_the_stream_and_fills_its_gaps() { + let mut color = cicp(1, 2, 1, None); + fill(&mut color, cicp(9, 1, 9, Some(true)).as_ref(), "test"); + let color = color.unwrap(); + assert_eq!(color.primaries, Some(1), "the stream wins a conflict"); + assert_eq!(color.matrix, Some(1)); + assert_eq!(color.transfer, Some(1), "the container fills an unspecified field"); + assert_eq!(color.full_range, Some(true), "the container fills an unsignalled range"); + } + + #[test] + fn fill_from_nothing_adds_nothing() { + let mut color = None; + fill(&mut color, None, "test"); + assert_eq!(color, None); + } + + #[test] + fn sei_reads_hdr10() { + assert_eq!(sei(&hdr10_sei()), hdr10()); + } + + #[test] + fn av1_metadata_reads_hdr10() { + let mut mdcv = vec![2]; + for v in [46_399u16, 19_136, 11_141, 52_232, 8_585, 3_014, 20_493, 21_561] { + mdcv.extend_from_slice(&v.to_be_bytes()); + } + mdcv.extend_from_slice(&(1000u32 << 8).to_be_bytes()); + mdcv.extend_from_slice(&82u32.to_be_bytes()); + let mastering = av1_metadata(&mdcv).mastering.unwrap(); + assert!((mastering.red.x - 0.708).abs() < 1e-4); + assert!((mastering.green.y - 0.797).abs() < 1e-4); + assert!((mastering.white.x - 0.3127).abs() < 1e-4); + assert_eq!(mastering.max_luminance, 1000.0); + assert!((mastering.min_luminance - 0.005).abs() < 1e-4); + + let cll = [1, 0x03, 0xE8, 0x01, 0x90]; + assert_eq!(av1_metadata(&cll), Hdr { content_light: hdr10().content_light, ..Default::default() }); + assert_eq!(av1_metadata(&[2, 0, 1]), Hdr::default(), "a short payload is ignored"); + } + + #[test] + fn sei_ignores_other_and_truncated_messages() { + // A user-data message, then a mastering display cut short. + let rbsp = [5, 2, 0xAA, 0xBB, 137, 24, 0, 1, 0x80]; + assert_eq!(sei(&rbsp), Hdr::default()); + } +} diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index f6fc67340b6..2937cbf5f76 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -13,7 +13,7 @@ use bytes::Bytes; -use super::{Error, NAL_TYPE_SPS, Sps}; +use super::{Error, NAL_TYPE_SEI, NAL_TYPE_SPS, Sps}; use crate::Result; use crate::catalog::hang::CatalogExt; use crate::codec::annexb::NalIterator; @@ -27,9 +27,9 @@ use crate::container::Frame; /// The catalog rendition fills in lazily once the codec config is known (avcC via /// [`initialize`](Self::initialize) for avc1, the first SPS for avc3). pub struct Import { - /// True for the avc1 shape: the codec config is out-of-band (avcC), so - /// keyframes are not scanned for an inline SPS. - avc1: bool, + /// The NALU length size for the avc1 shape, whose codec config is out-of-band (avcC), so + /// keyframes are not scanned for an inline SPS. `None` for Annex-B. + length_size: Option, track: crate::container::Producer, catalog: crate::codec::video::Catalog, last_sps: Option, @@ -52,7 +52,7 @@ impl Import { let catalog = crate::codec::video::Catalog::new(hint); let track = reserved.video(track, wire, None)?; let mut import = Self { - avc1: false, + length_size: None, track, catalog, last_sps: None, @@ -86,7 +86,7 @@ impl Import { // Only switch to avc1 mode once the avcC actually parses, so a parse failure leaves the // importer in avc3 mode where inline-SPS keyframes still self-initialize. let config = config_from_avcc(avcc_bytes)?; - self.avc1 = true; + self.length_size = Some(super::Avcc::parse(avcc_bytes)?.length_size); self.apply_config(config)?; Ok(()) } @@ -190,13 +190,18 @@ impl Import { /// keyframe's inline SPS and refining the catalog jitter as it goes. fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { for frame in frames { - // avc1 config arrives out-of-band via initialize(); avc3 carries SPS - // inline on keyframes. - if !self.avc1 - && frame.keyframe - && let Some(sps) = find_sps(&frame.payload) - { - self.configure_from_sps(&sps)?; + if frame.keyframe { + // HDR metadata first, so a keyframe that also configures publishes once. + let hdr = crate::codec::color::access_unit(&frame.payload, self.length_size, 1, |h| { + h & 0x1f == NAL_TYPE_SEI + }); + self.catalog.hdr(&mut self.track, hdr)?; + // avc1 config arrives out-of-band via initialize(); avc3 carries the SPS inline. + if self.length_size.is_none() + && let Some(sps) = find_sps(&frame.payload) + { + self.configure_from_sps(&sps)?; + } } if !self.catalog.configured() { @@ -249,6 +254,7 @@ fn config_from_avcc(avcc_bytes: &[u8]) -> Result { config.coded_width = avcc.coded_width; config.coded_height = avcc.coded_height; config.framerate = avcc.sps.first().and_then(|sps| super::sps_framerate(sps)); + config.color = avcc.sps.first().and_then(|sps| super::sps_color(sps)); config.description = Some(Bytes::copy_from_slice(avcc_bytes)); Ok(config) } @@ -267,6 +273,7 @@ fn config_from_sps(sps_nal: &[u8]) -> Result { config.coded_width = Some(sps.coded_width); config.coded_height = Some(sps.coded_height); config.framerate = super::sps_framerate(sps_nal); + config.color = super::sps_color(sps_nal); Ok(config) } diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index 634a2c2c21e..98292609969 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -21,6 +21,7 @@ pub use split::*; use bytes::{Buf, BufMut, Bytes, BytesMut}; // H.264 NAL unit types (ISO/IEC 14496-10 §7.4.1). +const NAL_TYPE_SEI: u8 = 6; const NAL_TYPE_SPS: u8 = 7; const NAL_TYPE_PPS: u8 = 8; @@ -175,8 +176,15 @@ pub(crate) fn sps_framerate(nal: &[u8]) -> Option { Some(scale as f64 / units as f64) } -/// What an SPS VUI declares about picture timing. +/// The colour an SPS NAL unit's VUI signals, when it carries a video signal type. +pub(crate) fn sps_color(nal: &[u8]) -> Option { + sps_vui(nal)?.color +} + +/// What an SPS VUI declares about picture timing and colour. struct Vui { + /// The video signal type's range and colour description, unspecified fields absent. + color: Option, /// `frame_mbs_only_flag`: every picture is a frame, so two ticks are one frame. frame_only: bool, /// One frame's duration as `(units, scale)` when `fixed_frame_rate_flag` is set. @@ -280,11 +288,15 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result> { if r.read_flag()? { r.skip_bits(1)?; // overscan_appropriate_flag } + let mut color = None; if r.read_flag()? { - r.skip_bits(4)?; // video_format, video_full_range_flag - if r.read_flag()? { - r.skip_bits(24)?; // colour_primaries, transfer_characteristics, matrix_coefficients - } + r.skip_bits(3)?; // video_format + let full_range = r.read_flag()?; + let (primaries, transfer, matrix) = match r.read_flag()? { + true => (r.read_u8()?, r.read_u8()?, r.read_u8()?), + false => (2, 2, 2), + }; + color = crate::codec::color::cicp(primaries, transfer, matrix, Some(full_range)); } if r.read_flag()? { read_ue(&mut r)?; // chroma_sample_loc_type_top_field @@ -314,6 +326,7 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result> { r.skip_bits(1)?; // pic_struct_present_flag if !r.read_flag()? { return Ok(Some(Vui { + color, frame_only, period, depth: None, @@ -326,6 +339,7 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result> { } let depth = read_ue(&mut r)?; // max_num_reorder_frames Ok(Some(Vui { + color, frame_only, period, depth: Some(depth), diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index 78e612f22a3..de3e45a210a 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -31,9 +31,9 @@ use crate::container::Frame; /// catalog rendition fills in lazily once the codec config is known (hvcC via /// [`initialize`](Self::initialize) for hvc1, the first SPS for hev1). pub struct Import { - /// True for the hvc1 shape: the codec config is out-of-band (hvcC), so - /// frame payloads are length-prefixed rather than Annex-B and are never scanned. - hvc1: bool, + /// The NALU length size for the hvc1 shape, whose codec config is out-of-band (hvcC), so + /// frame payloads are length-prefixed rather than Annex-B. `None` for Annex-B. + length_size: Option, track: crate::container::Producer, catalog: crate::codec::video::Catalog, last_sps: Option, @@ -56,7 +56,7 @@ impl Import { let catalog = crate::codec::video::Catalog::new(hint); let track = reserved.video(track, wire, None)?; let mut import = Self { - hvc1: false, + length_size: None, track, catalog, last_sps: None, @@ -90,7 +90,7 @@ impl Import { // Only switch to hvc1 mode once the hvcC actually parses, so a parse failure leaves the // importer in hev1 mode where inline-SPS keyframes still self-initialize. let config = super::config_from_hvcc(hvcc_bytes)?; - self.hvc1 = true; + self.length_size = Some(super::Hvcc::parse(hvcc_bytes)?.length_size); self.apply_config(config)?; Ok(()) } @@ -193,14 +193,20 @@ impl Import { /// keyframe's inline SPS and refining the catalog jitter as it goes. fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { for frame in frames { - // hvc1 config arrives out-of-band via initialize(); hev1 carries SPS inline on - // keyframes. Scanning an hvc1 payload would misread a length prefix as a start - // code (a 322-byte NAL prefixes as `00 00 01 42`, an SPS NAL header). - if !self.hvc1 - && frame.keyframe - && let Some(sps) = find_sps(&frame.payload) - { - self.configure_from_sps(&sps)?; + if frame.keyframe { + // HDR metadata first, so a keyframe that also configures publishes once. + let hdr = crate::codec::color::access_unit(&frame.payload, self.length_size, 2, |h| { + nal_unit_type(h) == scuffle_h265::NALUnitType::PrefixSeiNut + }); + self.catalog.hdr(&mut self.track, hdr)?; + // hvc1 config arrives out-of-band via initialize(); hev1 carries the SPS inline. + // Scanning an hvc1 payload as Annex-B would misread a length prefix as a start + // code (a 322-byte NAL prefixes as `00 00 01 42`, an SPS NAL header). + if self.length_size.is_none() + && let Some(sps) = find_sps(&frame.payload) + { + self.configure_from_sps(&sps)?; + } } // A keyframe we still can't configure (no SPS) is undecodable. @@ -237,7 +243,9 @@ pub fn config(buf: &[u8]) -> Result { /// The config an inline SPS describes (`in_band: true`, the hev1 shape). No `description`: the /// parameter sets ride with every keyframe. fn config_from_sps(sps_nal: &[u8]) -> Result { - let super::Sps { head, full } = super::Sps::parse(sps_nal)?; + let sps = super::Sps::parse(sps_nal)?; + let color = sps.color(); + let super::Sps { head, full } = sps; // The VUI (frame rate, aspect ratio) only comes with scuffle_h265's full parse. let vui_data = full .as_ref() @@ -263,6 +271,7 @@ fn config_from_sps(sps_nal: &[u8]) -> Result { let display_aspect = vui_data.display_aspect(head.width, head.height); config.display_aspect_width = display_aspect.map(|(width, _)| width); config.display_aspect_height = display_aspect.map(|(_, height)| height); + config.color = color; Ok(config) } diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index b9061aa2bdd..3a5a727d57b 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -199,7 +199,8 @@ impl Hvcc { pub(crate) fn config_from_hvcc(hvcc: &[u8]) -> Result { let params = Hvcc::parse(hvcc)?; let sps_nal = params.sps.first().ok_or(Error::MissingSps)?; - let sps = Sps::parse(sps_nal)?.head; + let parsed = Sps::parse(sps_nal)?; + let sps = parsed.head; let mut config = hang::catalog::VideoConfig::new(hang::catalog::H265 { in_band: false, @@ -212,10 +213,16 @@ pub(crate) fn config_from_hvcc(hvcc: &[u8]) -> Result Option { + Sps::parse(Hvcc::parse(hvcc).ok()?.sps.first()?).ok()?.color() +} + /// The reordering an SPS NAL unit declares for its highest sub-layer: `sps_max_num_reorder_pics`, /// and the picture period when the VUI's HRD parameters fix the picture rate within the CVS /// (ITU-T H.265 E.3.2). diff --git a/rs/moq-mux/src/codec/h265/sps.rs b/rs/moq-mux/src/codec/h265/sps.rs index f1fbaecc2d2..aca2eb0fb7d 100644 --- a/rs/moq-mux/src/codec/h265/sps.rs +++ b/rs/moq-mux/src/codec/h265/sps.rs @@ -57,6 +57,25 @@ impl Sps { } } +impl Sps { + /// The colour the VUI's video signal type carries, when scuffle_h265 accepted the SPS. + pub(crate) fn color(&self) -> Option { + let signal = &self.full.as_ref()?.vui_parameters.as_ref()?.video_signal_type; + // scuffle_h265 fills an absent video signal type with its default, inferred limited range + // included, so only a departure from it was signalled. A stream that signals limited range + // and nothing else reads as unsignalled, which a renderer assumes anyway. + if *signal == scuffle_h265::VideoSignalType::default() { + return None; + } + crate::codec::color::cicp( + signal.colour_primaries, + signal.transfer_characteristics, + signal.matrix_coeffs, + Some(signal.video_full_range_flag), + ) + } +} + impl SpsHead { fn from_rbsp(rbsp: &scuffle_h265::SpsRbsp) -> Result { let profile = &rbsp.profile_tier_level.general_profile; diff --git a/rs/moq-mux/src/codec/mod.rs b/rs/moq-mux/src/codec/mod.rs index 77d93c63e66..6cb47d8e532 100644 --- a/rs/moq-mux/src/codec/mod.rs +++ b/rs/moq-mux/src/codec/mod.rs @@ -10,6 +10,7 @@ pub mod aac; pub(crate) mod ac3; pub mod annexb; pub mod av1; +pub(crate) mod color; pub(crate) mod eac3; pub mod flac; pub mod h264; diff --git a/rs/moq-mux/src/codec/video.rs b/rs/moq-mux/src/codec/video.rs index 7a5a64fd672..e7cbf90ea68 100644 --- a/rs/moq-mux/src/codec/video.rs +++ b/rs/moq-mux/src/codec/video.rs @@ -58,12 +58,21 @@ pub(crate) struct Catalog { hint: VideoHint, /// The last config published, so an unchanged re-resolve doesn't re-mirror the rendition. last: Option, + /// The last config resolved, before the overlays, so new HDR metadata can republish it. + resolved: Option, + /// The static HDR metadata the bitstream carried in band, overlaid onto every config. + hdr: crate::codec::color::Hdr, } impl Catalog { /// Hold `hint` for every publish. pub(crate) fn new(hint: VideoHint) -> Self { - Self { hint, last: None } + Self { + hint, + last: None, + resolved: None, + hdr: Default::default(), + } } /// The config the hint alone resolves to, for importers that publish the catalog before parsing @@ -82,6 +91,8 @@ impl Catalog { /// publish. A changed config just re-mirrors the rendition; there are no fixed tracks to /// reject a reconfiguration. pub(crate) fn publish(&mut self, track: &mut Track, mut config: hang::catalog::VideoConfig) -> crate::Result<()> { + self.resolved = Some(config.clone()); + self.hdr.apply(&mut config); self.hint.apply(&mut config); if self.last.as_ref() == Some(&config) { return Ok(()); @@ -91,4 +102,18 @@ impl Catalog { self.last = Some(config); Ok(()) } + + /// Overlay the HDR metadata a keyframe carried in band, republishing the last config if it changed. + pub(crate) fn hdr(&mut self, track: &mut Track, hdr: crate::codec::color::Hdr) -> crate::Result<()> { + let mut next = self.hdr; + next.update(hdr); + if next == self.hdr { + return Ok(()); + } + self.hdr = next; + match self.resolved.clone() { + Some(config) => self.publish(track, config), + None => Ok(()), + } + } } diff --git a/rs/moq-mux/src/codec/vp9/import.rs b/rs/moq-mux/src/codec/vp9/import.rs index a9961b3f3ba..2b78901e75e 100644 --- a/rs/moq-mux/src/codec/vp9/import.rs +++ b/rs/moq-mux/src/codec/vp9/import.rs @@ -54,7 +54,8 @@ impl Import { } fn init(&mut self, vp9: hang::catalog::VP9, width: u16, height: u16) -> crate::Result<()> { - let mut config = hang::catalog::VideoConfig::new(vp9); + let mut config = hang::catalog::VideoConfig::new(vp9.clone()); + config.color = super::color(&vp9); config.coded_width = Some(width as u32); config.coded_height = Some(height as u32); diff --git a/rs/moq-mux/src/codec/vp9/mod.rs b/rs/moq-mux/src/codec/vp9/mod.rs index d2c773dc2d6..a3f10211f19 100644 --- a/rs/moq-mux/src/codec/vp9/mod.rs +++ b/rs/moq-mux/src/codec/vp9/mod.rs @@ -172,6 +172,17 @@ impl KeyFrame { } } +/// The neutral colour a VP9 codec string carries, for one read from a key frame or a `vpcC`, whose +/// every field is explicit. A placeholder codec string must not come through here. +pub(crate) fn color(vp9: &VP9) -> Option { + crate::codec::color::cicp( + vp9.color_primaries, + vp9.transfer_characteristics, + vp9.matrix_coefficients, + Some(vp9.full_range), + ) +} + /// Build a catalog [`VideoConfig`](hang::catalog::VideoConfig) from a VP9 frame, /// or `None` if the frame is not a key frame. /// @@ -184,7 +195,9 @@ pub(crate) fn config_from_keyframe(data: &[u8]) -> Result anyhow::Result { config.description = Some(Bytes::copy_from_slice(avcc_bytes)); config.coded_width = avcc.coded_width; config.coded_height = avcc.coded_height; + config.color = avcc.sps.first().and_then(|sps| crate::codec::h264::sps_color(sps)); Ok(config) } diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index d1fe0a9f13c..35a404594ad 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -462,23 +462,35 @@ impl Import { _ => return Err(Error::MultipleCodecs.into()), }; - let config = match codec { - mp4_atom::Codec::Avc1(avc1) => self.init_h264(false, &avc1.avcc, &avc1.visual, container)?, - mp4_atom::Codec::Avc3(avc3) => self.init_h264(true, &avc3.avcc, &avc3.visual, container)?, - mp4_atom::Codec::Hev1(hev1) => self.init_h265(true, &hev1.hvcc, &hev1.visual, container)?, - mp4_atom::Codec::Hvc1(hvc1) => self.init_h265(false, &hvc1.hvcc, &hvc1.visual, container)?, + let (mut config, colr) = match codec { + mp4_atom::Codec::Avc1(avc1) => ( + self.init_h264(false, &avc1.avcc, &avc1.visual, container)?, + avc1.colr.as_ref(), + ), + mp4_atom::Codec::Avc3(avc3) => ( + self.init_h264(true, &avc3.avcc, &avc3.visual, container)?, + avc3.colr.as_ref(), + ), + mp4_atom::Codec::Hev1(hev1) => ( + self.init_h265(true, &hev1.hvcc, &hev1.visual, container)?, + hev1.colr.as_ref(), + ), + mp4_atom::Codec::Hvc1(hvc1) => ( + self.init_h265(false, &hvc1.hvcc, &hvc1.visual, container)?, + hvc1.colr.as_ref(), + ), mp4_atom::Codec::Vp08(vp08) => { let mut config = VideoConfig::new(VideoCodec::VP8); config.coded_width = Some(vp08.visual.width as _); config.coded_height = Some(vp08.visual.height as _); config.container = container; - config + (config, vp08.colr.as_ref()) } mp4_atom::Codec::Vp09(vp09) => { // https://github.com/gpac/mp4box.js/blob/325741b592d910297bf609bc7c400fc76101077b/src/box-codecs.js#L238 let vpcc = &vp09.vpcc; - let mut config = VideoConfig::new(VP9 { + let vp9 = VP9 { profile: vpcc.profile, level: vpcc.level, bit_depth: vpcc.bit_depth, @@ -487,22 +499,27 @@ impl Import { transfer_characteristics: vpcc.transfer_characteristics, matrix_coefficients: vpcc.matrix_coefficients, full_range: vpcc.video_full_range_flag, - }); + }; + let mut config = VideoConfig::new(vp9.clone()); + config.color = crate::codec::vp9::color(&vp9); config.coded_width = Some(vp09.visual.width as _); config.coded_height = Some(vp09.visual.height as _); config.container = container; - config + (config, vp09.colr.as_ref()) } mp4_atom::Codec::Av01(av01) => { let mut config = VideoConfig::new(crate::codec::av1::av1_from_av1c(&av01.av1c)); + config.color = crate::codec::av1::color(&av01.av1c.config_obus); config.coded_width = Some(av01.visual.width as _); config.coded_height = Some(av01.visual.height as _); config.container = container; - config + (config, av01.colr.as_ref()) } mp4_atom::Codec::Unknown(unknown, _) => return Err(Error::UnknownCodec(*unknown).into()), unsupported => return Err(Error::UnsupportedCodec(Box::new(unsupported.clone())).into()), }; + // The codec config outranks the colr box a muxer added, which only fills its gaps. + crate::codec::color::fill(&mut config.color, colr.and_then(super::colr_color).as_ref(), "colr"); Ok(config) } @@ -527,6 +544,10 @@ impl Import { }); config.coded_width = Some(visual.width as _); config.coded_height = Some(visual.height as _); + config.color = avcc + .sequence_parameter_sets + .first() + .and_then(|sps| crate::codec::h264::sps_color(sps)); config.description = Some(description.freeze()); config.container = container; Ok(config) @@ -551,6 +572,7 @@ impl Import { level_idc: hvcc.general_level_idc, constraint_flags: hvcc.general_constraint_indicator_flags, }); + config.color = crate::codec::h265::hvcc_color(&description); config.description = Some(description.freeze()); config.coded_width = Some(visual.width as _); config.coded_height = Some(visual.height as _); diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index e28e27ced95..3a616fffebe 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -744,6 +744,43 @@ fn trun_duration(duration: Timestamp, timescale: moq_net::Timescale) -> Result Option { + let mp4_atom::Colr::Nclx { + colour_primaries, + transfer_characteristics, + matrix_coefficients, + full_range_flag, + } = *colr + else { + return None; + }; + // A code point past 255 is reserved in H.273, so it reads as unspecified. + let code = |value: u16| u8::try_from(value).unwrap_or(2); + crate::codec::color::cicp( + code(colour_primaries), + code(transfer_characteristics), + code(matrix_coefficients), + Some(full_range_flag), + ) +} + +/// The `nclx` box for a catalog colour, or `None` when it has no code point or range to carry. +/// An absent code point is written as unspecified (2) and an absent range as limited. +fn color_colr(color: Option<&hang::catalog::Color>) -> Option { + let color = color?; + if color.primaries.is_none() && color.transfer.is_none() && color.matrix.is_none() && color.full_range.is_none() { + return None; + } + Some(mp4_atom::Colr::Nclx { + colour_primaries: color.primaries.unwrap_or(2).into(), + transfer_characteristics: color.transfer.unwrap_or(2).into(), + matrix_coefficients: color.matrix.unwrap_or(2).into(), + full_range_flag: color.full_range.unwrap_or(false), + }) +} + pub(crate) fn synthesize_video_trak( track_id: u32, timescale: u64, @@ -782,6 +819,7 @@ pub(crate) fn synthesize_video_trak( // Codecs that carry an out-of-band config record require `description`. let require_description = || description.ok_or_else(|| Error::MissingVideoDescription(config.codec.to_string())); + let colr = color_colr(config.color.as_ref()); let sample_entry = match &config.codec { VideoCodec::H264(h264) => { let mut cursor = std::io::Cursor::new(require_description()?); @@ -792,12 +830,14 @@ pub(crate) fn synthesize_video_trak( mp4_atom::Codec::from(mp4_atom::Avc3 { visual, avcc, + colr, ..Default::default() }) } else { mp4_atom::Codec::from(mp4_atom::Avc1 { visual, avcc, + colr, ..Default::default() }) } @@ -810,12 +850,14 @@ pub(crate) fn synthesize_video_trak( mp4_atom::Codec::from(mp4_atom::Hev1 { visual, hvcc, + colr, ..Default::default() }) } else { mp4_atom::Codec::from(mp4_atom::Hvc1 { visual, hvcc, + colr, ..Default::default() }) } @@ -823,16 +865,19 @@ pub(crate) fn synthesize_video_trak( VideoCodec::AV1(av1) => mp4_atom::Codec::from(mp4_atom::Av01 { visual, av1c: crate::codec::av1::av1c_from_av1(av1), + colr, ..Default::default() }), VideoCodec::VP8 => mp4_atom::Codec::from(mp4_atom::Vp08 { visual, vpcc: crate::codec::vp8::vpcc(), + colr, ..Default::default() }), VideoCodec::VP9(vp9) => mp4_atom::Codec::from(mp4_atom::Vp09 { visual, vpcc: crate::codec::vp9::vpcc(vp9), + colr, ..Default::default() }), other => unreachable!("unsupported codecs rejected before geometry synthesis: {other:?}"), diff --git a/rs/moq-mux/src/container/mkv/export.rs b/rs/moq-mux/src/container/mkv/export.rs index 0b5530464fb..b43fd945152 100644 --- a/rs/moq-mux/src/container/mkv/export.rs +++ b/rs/moq-mux/src/container/mkv/export.rs @@ -708,6 +708,9 @@ fn build_video_track_entry( if let Some(h) = config.coded_height { video_children.push(MatroskaSpec::PixelHeight(h as u64)); } + if let Some(colour) = config.color.as_ref().and_then(matroska_colour) { + video_children.push(colour); + } let mut entry: Vec = vec![ MatroskaSpec::TrackNumber(track_number), @@ -725,6 +728,42 @@ fn build_video_track_entry( Ok(MatroskaSpec::TrackEntry(Master::Full(entry))) } +/// The Matroska `Colour` element for a catalog colour, carrying only the fields it sets. +fn matroska_colour(color: &hang::catalog::Color) -> Option { + let mut children = Vec::new(); + if let Some(v) = color.matrix { + children.push(MatroskaSpec::MatrixCoefficients(v.into())); + } + if let Some(full) = color.full_range { + children.push(MatroskaSpec::Range(if full { 2 } else { 1 })); + } + if let Some(v) = color.transfer { + children.push(MatroskaSpec::TransferCharacteristics(v.into())); + } + if let Some(v) = color.primaries { + children.push(MatroskaSpec::Primaries(v.into())); + } + if let Some(light) = color.content_light { + children.push(MatroskaSpec::MaxCLL(light.max_cll.into())); + children.push(MatroskaSpec::MaxFALL(light.max_fall.into())); + } + if let Some(m) = color.mastering { + children.push(MatroskaSpec::MasteringMetadata(Master::Full(vec![ + MatroskaSpec::PrimaryRChromaticityX(m.red.x), + MatroskaSpec::PrimaryRChromaticityY(m.red.y), + MatroskaSpec::PrimaryGChromaticityX(m.green.x), + MatroskaSpec::PrimaryGChromaticityY(m.green.y), + MatroskaSpec::PrimaryBChromaticityX(m.blue.x), + MatroskaSpec::PrimaryBChromaticityY(m.blue.y), + MatroskaSpec::WhitePointChromaticityX(m.white.x), + MatroskaSpec::WhitePointChromaticityY(m.white.y), + MatroskaSpec::LuminanceMax(m.max_luminance), + MatroskaSpec::LuminanceMin(m.min_luminance), + ]))); + } + (!children.is_empty()).then(|| MatroskaSpec::Colour(Master::Full(children))) +} + pub(super) fn build_audio_track_entry(track_number: u64, config: &AudioConfig) -> Result { let (codec_id, codec_private) = match &config.codec { AudioCodec::Opus => ( diff --git a/rs/moq-mux/src/container/mkv/import.rs b/rs/moq-mux/src/container/mkv/import.rs index 48d8891dd9c..8a87e522dfe 100644 --- a/rs/moq-mux/src/container/mkv/import.rs +++ b/rs/moq-mux/src/container/mkv/import.rs @@ -708,10 +708,84 @@ fn build_video_config( if config.coded_height.is_none() { config.coded_height = height; } + // The codec config outranks the Colour element a muxer added, which only fills its gaps. + let colour = video_children.and_then(|cs| { + cs.iter().find_map(|c| match c { + MatroskaSpec::Colour(Master::Full(colour)) => matroska_color(colour), + _ => None, + }) + }); + crate::codec::color::fill(&mut config.color, colour.as_ref(), "Colour"); Ok(config) } +/// The colour a Matroska `Colour` element signals. Its code points are H.273's, `Range` counts 0 +/// as unspecified, 1 as limited, and 2 as full, and a partial mastering display is dropped. +fn matroska_color(children: &[MatroskaSpec]) -> Option { + use hang::catalog::{Chromaticity, ContentLight, Mastering}; + + let (mut primaries, mut transfer, mut matrix, mut full_range) = (2, 2, 2, None); + let (mut max_cll, mut max_fall) = (None, None); + let mut mastering = None; + // A code point past 255 is reserved in H.273, so it reads as unspecified. + let code = |value: u64| u8::try_from(value).unwrap_or(2); + for child in children { + match child { + MatroskaSpec::Primaries(v) => primaries = code(*v), + MatroskaSpec::TransferCharacteristics(v) => transfer = code(*v), + MatroskaSpec::MatrixCoefficients(v) => matrix = code(*v), + MatroskaSpec::Range(1) => full_range = Some(false), + MatroskaSpec::Range(2) => full_range = Some(true), + MatroskaSpec::MaxCLL(v) => max_cll = Some(u16::try_from(*v).unwrap_or(u16::MAX)), + MatroskaSpec::MaxFALL(v) => max_fall = Some(u16::try_from(*v).unwrap_or(u16::MAX)), + MatroskaSpec::MasteringMetadata(Master::Full(fields)) => { + // Red, green, blue, and white x and y, then the max and min luminance. + let mut v = [None; 10]; + for field in fields { + let (i, value) = match field { + MatroskaSpec::PrimaryRChromaticityX(x) => (0, x), + MatroskaSpec::PrimaryRChromaticityY(x) => (1, x), + MatroskaSpec::PrimaryGChromaticityX(x) => (2, x), + MatroskaSpec::PrimaryGChromaticityY(x) => (3, x), + MatroskaSpec::PrimaryBChromaticityX(x) => (4, x), + MatroskaSpec::PrimaryBChromaticityY(x) => (5, x), + MatroskaSpec::WhitePointChromaticityX(x) => (6, x), + MatroskaSpec::WhitePointChromaticityY(x) => (7, x), + MatroskaSpec::LuminanceMax(x) => (8, x), + MatroskaSpec::LuminanceMin(x) => (9, x), + _ => continue, + }; + v[i] = Some(*value); + } + let xy = |i: usize| Some(Chromaticity { x: v[i]?, y: v[i + 1]? }); + mastering = (|| { + Some(Mastering { + red: xy(0)?, + green: xy(2)?, + blue: xy(4)?, + white: xy(6)?, + max_luminance: v[8]?, + min_luminance: v[9]?, + }) + })(); + } + _ => {} + } + } + + let mut color = crate::codec::color::cicp(primaries, transfer, matrix, full_range).unwrap_or_default(); + color.mastering = mastering; + if max_cll.is_some() || max_fall.is_some() { + color.content_light = Some(ContentLight { + max_cll: max_cll.unwrap_or(0), + max_fall: max_fall.unwrap_or(0), + }); + } + (!color.is_empty()).then_some(color) +} + + fn build_audio_config( codec_id: &str, codec_private: Option<&Bytes>, @@ -824,6 +898,7 @@ fn build_h264_config(codec_private: Option<&Bytes>) -> Result { config.description = Some(avcc_bytes.clone()); config.coded_width = avcc.coded_width; config.coded_height = avcc.coded_height; + config.color = avcc.sps.first().and_then(|sps| crate::codec::h264::sps_color(sps)); Ok(config) } @@ -847,6 +922,7 @@ fn build_h265_config(codec_private: Option<&Bytes>) -> Result { level_idc: hvcc.general_level_idc, constraint_flags: hvcc.general_constraint_indicator_flags, }); + config.color = crate::codec::h265::hvcc_color(&description); config.description = Some(description.freeze()); Ok(config) } @@ -863,6 +939,7 @@ fn build_av1_config(codec_private: Option<&Bytes>) -> Result { av1c.encode_body(&mut description)?; let mut config = VideoConfig::new(crate::codec::av1::av1_from_av1c(&av1c)); + config.color = crate::codec::av1::color(&av1c.config_obus); config.description = Some(description.freeze()); Ok(config) } From d94b321f356aa02f13952dce2ef94306e97867f8 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 20:43:10 -0700 Subject: [PATCH 03/14] test(moq-mux): cover each colour source, precedence, and the SDR round trip Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/codec/color.rs | 42 +++++----- rs/moq-mux/src/codec/h264/import.rs | 50 ++++++++++++ rs/moq-mux/src/codec/h264/mod.rs | 37 +++++++++ rs/moq-mux/src/codec/h265/import.rs | 34 +++++++++ rs/moq-mux/src/codec/h265/mod.rs | 36 +++++++++ rs/moq-mux/src/container/fmp4/import_test.rs | 80 ++++++++++++++++++++ rs/moq-mux/src/container/mkv/export.rs | 2 +- rs/moq-mux/src/container/mkv/import.rs | 2 +- rs/moq-mux/src/container/mkv/import_test.rs | 56 ++++++++++++++ 9 files changed, 320 insertions(+), 19 deletions(-) diff --git a/rs/moq-mux/src/codec/color.rs b/rs/moq-mux/src/codec/color.rs index 5ceb9b6b6c6..7d478f7a688 100644 --- a/rs/moq-mux/src/codec/color.rs +++ b/rs/moq-mux/src/codec/color.rs @@ -232,7 +232,7 @@ fn sei_mastering(p: &[u8]) -> Mastering { pub(crate) mod test { use super::*; - /// The HDR10 mastering display and light levels the fixtures below encode. + /// The HDR10 mastering display and light levels the x264 and x265 SEI fixtures carry. pub(crate) fn hdr10() -> Hdr { Hdr { mastering: Some(Mastering { @@ -250,21 +250,6 @@ pub(crate) mod test { } } - /// An SEI RBSP carrying [`hdr10`]: a mastering display then a content light level message. - pub(crate) fn hdr10_sei() -> Vec { - let mut rbsp = vec![137, 24]; - for v in [8500u16, 39850, 6550, 2300, 35400, 14600, 15635, 16450] { - rbsp.extend_from_slice(&v.to_be_bytes()); - } - rbsp.extend_from_slice(&10_000_000u32.to_be_bytes()); - rbsp.extend_from_slice(&50u32.to_be_bytes()); - rbsp.extend_from_slice(&[144, 4]); - rbsp.extend_from_slice(&1000u16.to_be_bytes()); - rbsp.extend_from_slice(&400u16.to_be_bytes()); - rbsp.push(0x80); - rbsp - } - #[test] fn cicp_drops_unspecified_and_reserved() { assert_eq!(cicp(2, 2, 2, None), None); @@ -296,7 +281,30 @@ pub(crate) mod test { #[test] fn sei_reads_hdr10() { - assert_eq!(sei(&hdr10_sei()), hdr10()); + use crate::codec::h264::fixtures::{SEI_CONTENT_LIGHT, SEI_MASTERING}; + let mut hdr = Hdr::default(); + for nal in [SEI_MASTERING, SEI_CONTENT_LIGHT] { + hdr.update(sei(&h264_parser::nal::ebsp_to_rbsp(&nal[1..]))); + } + assert_eq!(hdr, hdr10()); + } + + #[test] + fn access_unit_reads_both_shapes() { + use crate::codec::h264::fixtures::{PPS, SEI_CONTENT_LIGHT, SEI_MASTERING, SPS_HDR10}; + let is_sei = |h: u8| h & 0x1f == 6; + let nals = [SPS_HDR10, PPS, SEI_MASTERING, SEI_CONTENT_LIGHT]; + + let mut annexb = Vec::new(); + let mut prefixed = Vec::new(); + for nal in nals { + annexb.extend_from_slice(&[0, 0, 0, 1]); + annexb.extend_from_slice(nal); + prefixed.extend_from_slice(&(nal.len() as u32).to_be_bytes()); + prefixed.extend_from_slice(nal); + } + assert_eq!(access_unit(&annexb.into(), None, 1, is_sei), hdr10()); + assert_eq!(access_unit(&prefixed.into(), Some(4), 1, is_sei), hdr10()); } #[test] diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index 2937cbf5f76..b7c4f49987a 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -309,6 +309,56 @@ mod tests { (track, catalog) } + /// An HDR10 keyframe publishes its VUI colour and the static HDR metadata its SEI carries. + #[tokio::test(start_paused = true)] + async fn avc3_keyframe_publishes_hdr10_color() { + use crate::codec::h264::fixtures::{PPS, SEI_CONTENT_LIGHT, SEI_MASTERING, SPS_HDR10}; + + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; + let mut annexb = BytesMut::new(); + for nal in [SPS_HDR10, PPS, SEI_MASTERING, SEI_CONTENT_LIGHT, idr] { + annexb.extend_from_slice(&[0, 0, 0, 1]); + annexb.extend_from_slice(nal); + } + + let mut split = Split::new(); + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.reserve(), Default::default()).unwrap(); + let pts = moq_net::Timestamp::from_micros(0).unwrap(); + let mut frames = split.decode(&annexb, pts).expect("split keyframe"); + frames.extend(split.flush(pts).expect("flush keyframe")); + import.decode(frames).expect("decode keyframe"); + + let color = catalog.snapshot().video.renditions["video"].color.clone().expect("colour"); + assert_eq!( + (color.primaries, color.transfer, color.matrix, color.full_range), + (Some(9), Some(16), Some(9), Some(false)) + ); + let hdr = crate::codec::color::test::hdr10(); + assert_eq!((color.mastering, color.content_light), (hdr.mastering, hdr.content_light)); + } + + /// The stream outranks the caller's hint, which only fills what the stream leaves unsaid. + #[tokio::test(start_paused = true)] + async fn stream_color_outranks_the_hint() { + use crate::codec::h264::fixtures::{PPS, SPS_BT709}; + + let mut hinted = hang::catalog::Color::default(); + hinted.primaries = Some(9); + hinted.content_light = crate::codec::color::test::hdr10().content_light; + let mut hint = crate::catalog::VideoHint::default(); + hint.color = Some(hinted); + + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.reserve(), hint).unwrap(); + let avcc = crate::codec::h264::build_avcc(&[Bytes::from_static(SPS_BT709)], &[Bytes::from_static(PPS)]).unwrap(); + import.initialize(&avcc).unwrap(); + + let color = catalog.snapshot().video.renditions["video"].color.clone().expect("colour"); + assert_eq!(color.primaries, Some(1), "the SPS outranks the hint"); + assert_eq!(color.content_light, crate::codec::color::test::hdr10().content_light); + } + /// Only a fixed-rate SPS states the frame rate; without `fixed_frame_rate_flag` its tick is a /// ceiling, left for the catalog estimator to measure. #[test] diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index 98292609969..db342fc5bb1 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -768,12 +768,49 @@ pub(crate) mod fixtures { 0x03, 0x2e, 0x04, 0x00, 0x00, 0x89, 0x54, 0x40, 0x00, 0x11, 0x2a, 0x8e, 0x4c, 0x30, 0x07, 0x8b, 0x16, 0x89, ]; pub(crate) const PPS: &[u8] = &[0x68, 0xeb, 0xe3, 0xcb, 0x22, 0xc0]; + /// `colorprim=bt709:transfer=bt709:colormatrix=bt709:range=tv` on a Baseline encode: an SDR + /// video signal type with a full colour description. + pub(crate) const SPS_BT709: &[u8] = &[ + 0x67, 0x42, 0xc0, 0x0a, 0xda, 0x10, 0x9b, 0x01, 0x6a, 0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x03, 0x00, 0x80, + 0x00, 0x00, 0x19, 0x07, 0x89, 0x13, 0x50, + ]; + /// The same encode signalling BT.2020 primaries, PQ, and the BT.2020 non-constant matrix. + pub(crate) const SPS_HDR10: &[u8] = &[ + 0x67, 0x42, 0xc0, 0x0a, 0xda, 0x10, 0x9b, 0x01, 0x6a, 0x12, 0x20, 0x12, 0x80, 0x00, 0x00, 0x03, 0x00, 0x80, + 0x00, 0x00, 0x19, 0x07, 0x89, 0x13, 0x50, + ]; + /// x264's `mastering-display` SEI for a 1000-nit BT.2020 display with a D65 white point; its + /// minimum luminance needs emulation prevention. + pub(crate) const SEI_MASTERING: &[u8] = &[ + 0x06, 0x89, 0x18, 0x21, 0x34, 0x9b, 0xaa, 0x19, 0x96, 0x08, 0xfc, 0x8a, 0x48, 0x39, 0x08, 0x3d, 0x13, 0x40, + 0x42, 0x00, 0x98, 0x96, 0x80, 0x00, 0x00, 0x03, 0x00, 0x32, 0x80, + ]; + /// x264's `cll=1000,400` SEI. + pub(crate) const SEI_CONTENT_LIGHT: &[u8] = &[ + 0x06, 0x90, 0x04, 0x03, 0xe8, 0x01, 0x90, 0x80, + ]; } #[cfg(test)] mod tests { use super::*; + /// Only a VUI with a video signal type adds colour, and its colour description is read whole. + #[test] + fn sps_color_reads_the_video_signal_type() { + let sdr = sps_color(fixtures::SPS_BT709).unwrap(); + assert_eq!( + (sdr.primaries, sdr.transfer, sdr.matrix, sdr.full_range), + (Some(1), Some(1), Some(1), Some(false)) + ); + let hdr = sps_color(fixtures::SPS_HDR10).unwrap(); + assert_eq!( + (hdr.primaries, hdr.transfer, hdr.matrix, hdr.full_range), + (Some(9), Some(16), Some(9), Some(false)) + ); + assert_eq!(sps_color(fixtures::SPS_IPB), None, "a VUI without a video signal type"); + } + const SC4: &[u8] = &[0, 0, 0, 1]; #[test] diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index de3e45a210a..20919dc52c1 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -373,6 +373,40 @@ mod tests { use super::*; use crate::codec::h265::{Split, fixtures}; + /// An HDR10 keyframe publishes its VUI colour and the static HDR metadata its SEI carries. + #[tokio::test(start_paused = true)] + async fn hev1_keyframe_publishes_hdr10_color() { + let idr: &[u8] = &[0x26, 0x01, 0x80, 0xaa]; // IdrWRadl (19) + let mut annexb = BytesMut::new(); + for nal in [ + fixtures::VPS, + fixtures::SPS_HDR10, + fixtures::PPS, + fixtures::SEI_CONTENT_LIGHT, + fixtures::SEI_MASTERING, + idr, + ] { + annexb.extend_from_slice(&[0, 0, 0, 1]); + annexb.extend_from_slice(nal); + } + + let mut split = Split::new(); + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.reserve(), Default::default()).unwrap(); + let pts = moq_net::Timestamp::from_micros(0).unwrap(); + let mut frames = split.decode(&annexb, pts).expect("split keyframe"); + frames.extend(split.flush(pts).expect("flush keyframe")); + import.decode(frames).expect("decode keyframe"); + + let color = catalog.snapshot().video.renditions["video"].color.clone().expect("colour"); + assert_eq!( + (color.primaries, color.transfer, color.matrix, color.full_range), + (Some(9), Some(16), Some(9), Some(false)) + ); + let hdr = crate::codec::color::test::hdr10(); + assert_eq!((color.mastering, color.content_light), (hdr.mastering, hdr.content_light)); + } + fn setup(name: &str) -> (moq_net::track::Producer, crate::catalog::Producer) { let mut broadcast = moq_net::broadcast::Info::new().produce(); let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index 3a5a727d57b..88a6741751f 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -569,6 +569,22 @@ pub(crate) mod fixtures { 0x02, 0x00, 0x00, 0x03, 0x00, 0x3c, 0x10, ]; pub(crate) const PPS: &[u8] = &[0x44, 0x01, 0xc1, 0x72, 0xb4, 0x62, 0x40]; + + /// SPS from a 64x64 Main 10 x265 encode with `colorprim=bt2020:transfer=smpte2084:colormatrix=bt2020nc`. + pub(crate) const SPS_HDR10: &[u8] = &[ + 0x42, 0x01, 0x01, 0x02, 0x20, 0x00, 0x00, 0x03, 0x00, 0x90, 0x00, 0x00, 0x03, 0x00, 0x00, 0x03, 0x00, 0x1e, + 0xa0, 0x20, 0x81, 0x04, 0xd9, 0x65, 0x65, 0x4a, 0x4c, 0x2f, 0x01, 0x6a, 0x12, 0x20, 0x12, 0x08, 0x00, 0x00, + 0x03, 0x00, 0x08, 0x00, 0x00, 0x03, 0x00, 0xc8, 0x40, + ]; + /// The same encode's `max-cll=1000,400` prefix SEI. + pub(crate) const SEI_CONTENT_LIGHT: &[u8] = &[ + 0x4e, 0x01, 0x90, 0x04, 0x03, 0xe8, 0x01, 0x90, 0x80, + ]; + /// The same encode's `master-display` prefix SEI: a 1000-nit BT.2020 display with a D65 white point. + pub(crate) const SEI_MASTERING: &[u8] = &[ + 0x4e, 0x01, 0x89, 0x18, 0x21, 0x34, 0x9b, 0xaa, 0x19, 0x96, 0x08, 0xfc, 0x8a, 0x48, 0x39, 0x08, 0x3d, 0x13, + 0x40, 0x42, 0x00, 0x98, 0x96, 0x80, 0x00, 0x00, 0x03, 0x00, 0x32, 0x80, + ]; /// SPS from a 64x64, 25 fps x265 encode with `bframes=3:b-pyramid=1` and HRD signalling /// (`sps_max_num_reorder_pics` 2). pub(crate) const SPS_PYRAMID: &[u8] = &[ @@ -689,6 +705,26 @@ mod tests { /// The real-SPS fixture resolves a full out-of-band config: dimensions from /// the parsed SPS and the hvcC itself as the catalog `description`. + /// The hvcC's SPS signals the colour; a refused SPS signals none rather than failing. + #[test] + fn hvcc_color_reads_the_sps_vui() { + let hvcc = |sps: &'static [u8]| { + build_hvcc( + &[Bytes::from_static(fixtures::VPS)], + &[Bytes::from_static(sps)], + &[Bytes::from_static(fixtures::PPS)], + ) + .unwrap() + }; + let color = config_from_hvcc(&hvcc(fixtures::SPS_HDR10)).unwrap().color.unwrap(); + assert_eq!( + (color.primaries, color.transfer, color.matrix, color.full_range), + (Some(9), Some(16), Some(9), Some(false)) + ); + assert_eq!(hvcc_color(&hvcc(fixtures::SPS)), None); + assert_eq!(hvcc_color(&hvcc(fixtures::SPS_ZERO_COLOUR)), None); + } + #[test] fn config_from_hvcc_resolves_real_sps() { let hvcc = fixtures::hvcc(); diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 02a88e0e6d0..225c0fa514c 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -269,6 +269,9 @@ fn test_av1_catalog() { let video = catalog.video.renditions.values().next().unwrap(); assert!(video.codec.to_string().starts_with("av01."), "codec: {}", video.codec); + // The av1C's sequence header leaves the code points unspecified, but AV1 always states the range. + let color = video.color.as_ref().expect("the range"); + assert_eq!((color.primaries, color.full_range), (None, Some(false))); assert!(matches!(video.container, Container::Cmaf { .. })); let Container::Cmaf { init, .. } = &video.container else { @@ -2231,4 +2234,81 @@ async fn ttml_body_begins_at_the_serialized_frame_timestamp() { payload.contains(r#""#), "{payload}" ); + +/// An avc1 init whose avcC carries `sps` and whose sample entry carries the `colr` box for `color`. +fn avc1_init(sps: &'static [u8], color: Option) -> bytes::Bytes { + use crate::codec::h264::fixtures::PPS; + use bytes::Bytes; + use hang::catalog::{H264, VideoConfig}; + + let avcc = crate::codec::h264::build_avcc(&[Bytes::from_static(sps)], &[Bytes::from_static(PPS)]).unwrap(); + let mut config = VideoConfig::new(H264 { + profile: 0x42, + constraints: 0xc0, + level: 0x0a, + inline: false, + }); + config.coded_width = Some(64); + config.coded_height = Some(64); + config.color = color; + let trak = super::synthesize_video_trak(1, 90_000, &config, Some(&avcc)).unwrap(); + let trex = mp4_atom::Trex { + track_id: 1, + default_sample_description_index: 1, + ..Default::default() + }; + super::encode_init(None, vec![trak], vec![trex]).unwrap() +} + +fn color(primaries: u8, transfer: u8, matrix: u8, full_range: bool) -> hang::catalog::Color { + crate::codec::color::cicp(primaries, transfer, matrix, Some(full_range)).unwrap() +} + +/// A colr box the SPS disagrees with loses to the SPS: the encoder signalled it. +#[test] +fn sps_color_outranks_colr() { + let catalog = run_fmp4(&avc1_init(crate::codec::h264::fixtures::SPS_BT709, Some(color(9, 16, 9, true)))); + let video = catalog.video.renditions.values().next().unwrap(); + assert_eq!(video.color, Some(color(1, 1, 1, false))); +} + +/// A colr box fills what the SPS leaves unspecified. +#[test] +fn colr_fills_what_the_sps_leaves_unsaid() { + let catalog = run_fmp4(&avc1_init(crate::codec::h264::fixtures::SPS_IPB, Some(color(9, 16, 9, true)))); + let video = catalog.video.renditions.values().next().unwrap(); + assert_eq!(video.color, Some(color(9, 16, 9, true))); +} + +/// A source that signals no colour adds no catalog field and no colr box. +#[test] +fn unsignalled_color_stays_absent() { + let init = avc1_init(crate::codec::h264::fixtures::SPS_IPB, None); + let (_, moov) = decode_init(&init); + let mp4_atom::Codec::Avc1(avc1) = &moov.trak[0].mdia.minf.stbl.stsd.codecs[0] else { + panic!("expected avc1"); + }; + assert_eq!(avc1.colr, None); + + let catalog = run_fmp4(&init); + let video = catalog.video.renditions.values().next().unwrap(); + assert_eq!(video.color, None); + assert!(serde_json::to_value(video).unwrap().get("color").is_none()); +} + +/// An SDR init imported to the catalog and exported again is byte-identical. +#[test] +fn sdr_color_round_trips_byte_identical() { + let init = avc1_init(crate::codec::h264::fixtures::SPS_BT709, Some(color(1, 1, 1, false))); + let catalog = run_fmp4(&init); + let video = catalog.video.renditions.values().next().unwrap(); + assert_eq!(video.color, Some(color(1, 1, 1, false))); + + let trak = super::synthesize_video_trak(1, 90_000, video, video.description.as_deref()).unwrap(); + let trex = mp4_atom::Trex { + track_id: 1, + default_sample_description_index: 1, + ..Default::default() + }; + assert_eq!(super::encode_init(None, vec![trak], vec![trex]).unwrap(), init); } diff --git a/rs/moq-mux/src/container/mkv/export.rs b/rs/moq-mux/src/container/mkv/export.rs index b43fd945152..94b7fb514db 100644 --- a/rs/moq-mux/src/container/mkv/export.rs +++ b/rs/moq-mux/src/container/mkv/export.rs @@ -729,7 +729,7 @@ fn build_video_track_entry( } /// The Matroska `Colour` element for a catalog colour, carrying only the fields it sets. -fn matroska_colour(color: &hang::catalog::Color) -> Option { +pub(super) fn matroska_colour(color: &hang::catalog::Color) -> Option { let mut children = Vec::new(); if let Some(v) = color.matrix { children.push(MatroskaSpec::MatrixCoefficients(v.into())); diff --git a/rs/moq-mux/src/container/mkv/import.rs b/rs/moq-mux/src/container/mkv/import.rs index 8a87e522dfe..c17d445927b 100644 --- a/rs/moq-mux/src/container/mkv/import.rs +++ b/rs/moq-mux/src/container/mkv/import.rs @@ -722,7 +722,7 @@ fn build_video_config( /// The colour a Matroska `Colour` element signals. Its code points are H.273's, `Range` counts 0 /// as unspecified, 1 as limited, and 2 as full, and a partial mastering display is dropped. -fn matroska_color(children: &[MatroskaSpec]) -> Option { +pub(super) fn matroska_color(children: &[MatroskaSpec]) -> Option { use hang::catalog::{Chromaticity, ContentLight, Mastering}; let (mut primaries, mut transfer, mut matrix, mut full_range) = (2, 2, 2, None); diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index a5d692df88a..33aab79db5b 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -1106,3 +1106,59 @@ fn subtitle_codec_state_is_refused() { "{result:?}" ); } + +/// The SPS outranks the Colour element, which still fills what the SPS leaves unsaid: here the +/// static HDR metadata. +#[test] +fn sps_color_outranks_colour_which_fills_its_gaps() { + use crate::codec::h264::fixtures::{PPS, SPS_BT709}; + + let hdr = crate::codec::color::test::hdr10(); + let mut container = crate::codec::color::cicp(9, 16, 9, Some(true)).unwrap(); + container.mastering = hdr.mastering; + container.content_light = hdr.content_light; + let colour = super::export::matroska_colour(&container).unwrap(); + + let avcc = crate::codec::h264::build_avcc(&[bytes::Bytes::from_static(SPS_BT709)], &[bytes::Bytes::from_static(PPS)]).unwrap(); + let entry = MatroskaSpec::TrackEntry(Master::Full(vec![ + MatroskaSpec::TrackNumber(1), + MatroskaSpec::TrackUID(1), + MatroskaSpec::TrackType(1), + MatroskaSpec::CodecID("V_MPEG4/ISO/AVC".to_string()), + MatroskaSpec::CodecPrivate(avcc.to_vec()), + MatroskaSpec::Video(Master::Full(vec![ + MatroskaSpec::PixelWidth(64), + MatroskaSpec::PixelHeight(64), + colour, + ])), + ])); + let data = MkvBuilder::new() + .header("matroska") + .segment_start() + .info(1_000_000) + .tracks(vec![entry]) + .segment_end() + .build(); + + let catalog = run(&data); + let color = catalog.video.renditions.values().next().unwrap().color.clone().unwrap(); + assert_eq!( + (color.primaries, color.transfer, color.matrix, color.full_range), + (Some(1), Some(1), Some(1), Some(false)) + ); + assert_eq!((color.mastering, color.content_light), (hdr.mastering, hdr.content_light)); +} + +/// Every catalog colour field survives a Matroska Colour element. +#[test] +fn colour_element_round_trips() { + let hdr = crate::codec::color::test::hdr10(); + let mut color = crate::codec::color::cicp(9, 18, 9, Some(true)).unwrap(); + color.mastering = hdr.mastering; + color.content_light = hdr.content_light; + let MatroskaSpec::Colour(Master::Full(children)) = super::export::matroska_colour(&color).unwrap() else { + panic!("expected a full Colour element"); + }; + assert_eq!(super::import::matroska_color(&children), Some(color)); + assert_eq!(super::export::matroska_colour(&hang::catalog::Color::default()), None); +} From a799439adea5ec4f8b697868d5cb05f39a9f6cd7 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 20:45:56 -0700 Subject: [PATCH 04/14] feat(hang): mirror the colour model in js/hang, the draft, and the docs Co-Authored-By: Claude Opus 5.5 --- doc/concept/hang.md | 1 + drafts/draft-lcurley-moq-hang.md | 55 +++++++++++++++++++++++++++++++ js/hang/src/catalog/video.test.ts | 29 ++++++++++++++++ js/hang/src/catalog/video.ts | 42 ++++++++++++++++++++++- 4 files changed, 126 insertions(+), 1 deletion(-) diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 655ea2e417d..618d872b683 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -67,6 +67,7 @@ A few things the catalog can express beyond decoder config: - **Track names.** The map key is a rendition's ID and, by default, its track name. A rendition sets `track` only when the two must differ, such as two renditions of same-named tracks in different broadcasts, or a rendition whose media changed and so needs a new name; readers subscribe to `track` when present. An ID can't name both a video and an audio rendition. - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. +- **Colour.** A video rendition may carry `color`: the H.273 primaries, transfer, matrix, and range, plus the HDR10 mastering display and content light levels, so a renderer can set up HDR without parsing the bitstream. The Rust importers fill it from the SPS VUI, AV1 sequence header, and VP9 header, then fill gaps from a container's `colr` box or Matroska `Colour` element; the stream wins a disagreement. The HDR10 values come from H.264 and H.265 SEI or AV1 metadata OBUs in MPEG-TS and raw elementary streams, or from Matroska, but not yet from fMP4 or FLV. Nothing in `@moq/watch` reads it yet. - **Disabled renditions.** An audio or video rendition with `enabled: false` has no frames coming, and a player must not select it. Pausing a rendition in `@moq/publish` keeps it in the catalog this way rather than removing it, so a mute is a one-field update; a disabled video rendition encodes one black keyframe first, so a player that predates the field shows black instead of a frozen picture. The field is written only when false. The older video `stalled` field is no longer written, and readers ignore it. - **Preference.** A video rendition may carry a signed `preference`, default `0`. A player keeps only the highest preference it can decode, then picks by size and bitrate within it, as do the single-rendition egresses (RTMP play, WHEP, FLV, the transcoder's source). A publisher that adds its own H.264 copy of an H.265 source marks the copy `-1`, so only a viewer that can't decode H.265 pulls it. Picking a rendition by name still overrides it. The field is written only when nonzero. - **Archive.** A broadcast may advertise an `archive` entry mapping each track, the catalog included, to its own timeline track (a small index of the track's spans: timing plus group and frame positions) and fixed nominal record duration, plus the replay MoQ path, object-store URL, and format version when recorded. Media records end at the first group boundary at or beyond a 4s target; sparse timelines such as the catalog declare zero. Tracks commit and expire independently. An untimed track has no timing to index, so its timeline declares no duration and holds one record per group: it can be recorded and replayed, but not turned into HLS or kept in a DVR. The timelines are what let the [HLS gateway](/bin/hls) build playlists without subscribing to media. A reader that cannot decode the `archive` entry, such as one from a different version, ignores it and loses the timelines and replay, but live media still plays. diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index aae676df53f..a8eb7780c3a 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -35,6 +35,28 @@ normative: date: false informative: + h273: + title: "Coding-independent code points for video signal type identification" + target: https://www.itu.int/rec/T-REC-H.273 + author: + - org: ITU-T + seriesinfo: + ITU-T Recommendation: H.273 + date: false + st2086: + title: "Mastering Display Color Volume Metadata Supporting High Luminance and Wide Color Gamut Images" + author: + - org: SMPTE + seriesinfo: + SMPTE: ST 2086 + date: false + cta861: + title: "HDR Static Metadata Extensions" + author: + - org: CTA + seriesinfo: + CTA: 861.3 + date: false --- abstract @@ -208,6 +230,7 @@ type VideoDecoderConfigExtensions = { "displayAspectWidth": number | undefined, "displayAspectHeight": number | undefined, "preference": number | undefined, + "color": Color | undefined, } ~~~ @@ -222,6 +245,37 @@ A consumer selecting a rendition automatically MUST choose among the enabled ({{ A consumer MAY select another rendition at the viewer's explicit request. A consumer that carries every rendition, rather than selecting one, ignores the field. +`color` describes the decoded pixels so a renderer can configure its pipeline without parsing the bitstream: + +~~~ +type Color = { + "primaries": number | undefined, + "transfer": number | undefined, + "matrix": number | undefined, + "fullRange": boolean | undefined, + "mastering": { + "red": Chromaticity, + "green": Chromaticity, + "blue": Chromaticity, + "white": Chromaticity, + "maxLuminance": number, + "minLuminance": number, + } | undefined, + "contentLight": { + "maxCll": number, + "maxFall": number, + } | undefined, +} + +type Chromaticity = { "x": number, "y": number } +~~~ + +`primaries`, `transfer`, and `matrix` are the ColourPrimaries, TransferCharacteristics, and MatrixCoefficients code points of {{h273}}, and `fullRange` is its VideoFullRangeFlag. +`mastering` is the {{st2086}} mastering display: CIE 1931 xy chromaticities and luminance in candelas per square metre. +`contentLight` is the {{cta861}} MaxCLL and MaxFALL in candelas per square metre, where 0 means unknown. +A publisher MUST omit a field its source leaves unspecified rather than write a default, and SHOULD prefer the value signalled in the elementary stream over a container's when they disagree. +`color` is not part of the track's identity ({{identity}}). + For example: ~~~ @@ -1278,6 +1332,7 @@ This document has no IANA actions. - Audio `sampleRate` and `numberOfChannels` describe the decoded output, so HE-AAC names its SBR rate and parametric stereo names two channels. - Added an optional `track` field to video, audio, text, `json`, and `binary` entries naming the track in its broadcast; the entry's key is its ID and names the track when `track` is absent. A consumer MUST reject a catalog using one ID for both a video and an audio rendition. - Defined video `displayAspectWidth` and `displayAspectHeight` as the whole picture's display aspect, not the sample aspect. +- Added the optional video `color` field: H.273 primaries, transfer, matrix, and range, plus the SMPTE ST 2086 mastering display and CTA-861.3 content light levels. ## moq-hang-03 {:numbered="false"} diff --git a/js/hang/src/catalog/video.test.ts b/js/hang/src/catalog/video.test.ts index 5ccaeffce13..0cc5899955b 100644 --- a/js/hang/src/catalog/video.test.ts +++ b/js/hang/src/catalog/video.test.ts @@ -166,3 +166,32 @@ test("video warmup round trips as optional integer milliseconds", () => { expect(() => VideoConfigSchema.parse({ ...base, warmup })).toThrow(); } }); + +test("color parses the H.273 code points and HDR10 metadata a Rust publisher writes", () => { + const color = { + primaries: 9, + transfer: 16, + matrix: 9, + fullRange: false, + mastering: { + red: { x: 0.708, y: 0.292 }, + green: { x: 0.17, y: 0.797 }, + blue: { x: 0.131, y: 0.046 }, + white: { x: 0.3127, y: 0.329 }, + maxLuminance: 1000, + minLuminance: 0.005, + }, + contentLight: { maxCll: 1000, maxFall: 400 }, + }; + const config = VideoConfigSchema.parse({ codec: "hvc1.2.4.L93.B0", container: { kind: "legacy" }, color }); + expect(config.color).toEqual(color as typeof config.color); + expect(VideoConfigSchema.parse({ codec: "vp8", container: { kind: "legacy" } }).color).toBeUndefined(); +}); + +test("color refuses out-of-range code points and light levels", () => { + const parse = (color: unknown) => + VideoConfigSchema.safeParse({ codec: "vp8", container: { kind: "legacy" }, color }).success; + expect(parse({ primaries: 256 })).toBe(false); + expect(parse({ contentLight: { maxCll: 65536, maxFall: 0 } })).toBe(false); + expect(parse({ matrix: 0 })).toBe(true); +}); diff --git a/js/hang/src/catalog/video.ts b/js/hang/src/catalog/video.ts index f0297856c09..e2e01201ab3 100644 --- a/js/hang/src/catalog/video.ts +++ b/js/hang/src/catalog/video.ts @@ -2,7 +2,7 @@ import * as z from "@zod/mini"; import { compareName, compareNumber } from "./compare"; import { ContainerSchema } from "./container"; import { hexSchema } from "./hex"; -import { u53Schema } from "./integers"; +import { u8Schema, u53Schema } from "./integers"; import { RelativeBroadcastSchema } from "./path"; // Backwards compatibility: old track schema @@ -10,6 +10,43 @@ const TrackSchema = z.object({ name: z.string(), }); +// A CIE 1931 xy chromaticity coordinate. +const ChromaticitySchema = z.object({ x: z.number(), y: z.number() }); + +// A CTA-861.3 light level in candelas per square metre; zero means unknown. +const lightSchema = z.number().check(z.int(), z.nonnegative(), z.lte(0xffff)); + +/** + * Schema for the colour a rendition's pixels carry. The code points are ITU-T H.273's, which every + * codec and container shares; an absent field was left unspecified by the source. + */ +export const ColorSchema = z.object({ + // H.273 ColourPrimaries: 1 is BT.709, 9 is BT.2020. + primaries: z.optional(u8Schema), + // H.273 TransferCharacteristics: 1 is BT.709, 16 is PQ, 18 is HLG. + transfer: z.optional(u8Schema), + // H.273 MatrixCoefficients: 0 is RGB, 1 is BT.709, 9 is BT.2020 non-constant luminance. + matrix: z.optional(u8Schema), + // True for full-range samples, false for limited (studio) range. + fullRange: z.optional(z.boolean()), + // The display the content was mastered on, as SMPTE ST 2086, luminance in cd/m². + mastering: z.optional( + z.object({ + red: ChromaticitySchema, + green: ChromaticitySchema, + blue: ChromaticitySchema, + white: ChromaticitySchema, + maxLuminance: z.number(), + minLuminance: z.number(), + }), + ), + // The content's light levels, as CTA-861.3. + contentLight: z.optional(z.object({ maxCll: lightSchema, maxFall: lightSchema })), +}); + +/** The colour a rendition's pixels carry: H.273 code points plus static HDR metadata. */ +export type Color = z.infer; + /** Schema for a single video rendition's decoder config. Mirrors WebCodecs VideoDecoderConfig. */ export const VideoConfigSchema = z.object({ // Optional reference to another broadcast that publishes this track, expressed @@ -44,6 +81,9 @@ export const VideoConfigSchema = z.object({ displayAspectWidth: z.optional(u53Schema), displayAspectHeight: z.optional(u53Schema), + // The colour the decoded pixels carry, so a renderer need not guess or reparse the bitstream. + color: z.optional(ColorSchema), + // The maximum frame rate of the video in frames per second. framerate: z.optional(z.number()), From 5ef93c49e1e991f6120c0e36c07819302c52798a Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 21:05:14 -0700 Subject: [PATCH 05/14] fix(moq-video): carry the probed colour into the importer hint Co-Authored-By: Claude Opus 5.5 --- rs/hang/src/catalog/video/color.rs | 5 +++- rs/moq-mux/src/codec/color.rs | 30 +++++++++++++++++--- rs/moq-mux/src/codec/h264/import.rs | 29 +++++++++++++------ rs/moq-mux/src/codec/h264/mod.rs | 4 +-- rs/moq-mux/src/codec/h265/import.rs | 10 +++++-- rs/moq-mux/src/codec/h265/mod.rs | 4 +-- rs/moq-mux/src/container/fmp4/import_test.rs | 10 +++++-- rs/moq-mux/src/container/mkv/export.rs | 2 +- rs/moq-mux/src/container/mkv/import.rs | 1 - rs/moq-mux/src/container/mkv/import_test.rs | 11 +++++-- rs/moq-video/src/encode/producer.rs | 1 + 11 files changed, 79 insertions(+), 28 deletions(-) diff --git a/rs/hang/src/catalog/video/color.rs b/rs/hang/src/catalog/video/color.rs index a539d153612..9bc61fe197d 100644 --- a/rs/hang/src/catalog/video/color.rs +++ b/rs/hang/src/catalog/video/color.rs @@ -126,7 +126,10 @@ mod test { matrix: Some(1), ..Default::default() }; - assert_eq!(serde_json::to_value(&color).unwrap(), serde_json::json!({ "matrix": 1 })); + assert_eq!( + serde_json::to_value(&color).unwrap(), + serde_json::json!({ "matrix": 1 }) + ); assert!(!color.is_empty()); assert!(Color::default().is_empty()); } diff --git a/rs/moq-mux/src/codec/color.rs b/rs/moq-mux/src/codec/color.rs index 7d478f7a688..e6c2f8af220 100644 --- a/rs/moq-mux/src/codec/color.rs +++ b/rs/moq-mux/src/codec/color.rs @@ -33,11 +33,22 @@ pub(crate) fn fill(color: &mut Option, fallback: Option<&Color>, source: return; }; let color = color.get_or_insert_with(Color::default); - fn field(name: &str, source: &str, slot: &mut Option, other: &Option) { + fn field( + name: &str, + source: &str, + slot: &mut Option, + other: &Option, + ) { match (&*slot, other) { (None, _) => *slot = other.clone(), (Some(kept), Some(other)) if kept != other => { - tracing::warn!(field = name, ?kept, ?other, source, "colour disagrees with the stream; keeping the stream's"); + tracing::warn!( + field = name, + ?kept, + ?other, + source, + "colour disagrees with the stream; keeping the stream's" + ); } _ => {} } @@ -47,7 +58,12 @@ pub(crate) fn fill(color: &mut Option, fallback: Option<&Color>, source: field("matrix", source, &mut color.matrix, &fallback.matrix); field("full_range", source, &mut color.full_range, &fallback.full_range); field("mastering", source, &mut color.mastering, &fallback.mastering); - field("content_light", source, &mut color.content_light, &fallback.content_light); + field( + "content_light", + source, + &mut color.content_light, + &fallback.content_light, + ); } /// The static HDR metadata an access unit carries in band. @@ -323,7 +339,13 @@ pub(crate) mod test { assert!((mastering.min_luminance - 0.005).abs() < 1e-4); let cll = [1, 0x03, 0xE8, 0x01, 0x90]; - assert_eq!(av1_metadata(&cll), Hdr { content_light: hdr10().content_light, ..Default::default() }); + assert_eq!( + av1_metadata(&cll), + Hdr { + content_light: hdr10().content_light, + ..Default::default() + } + ); assert_eq!(av1_metadata(&[2, 0, 1]), Hdr::default(), "a short payload is ignored"); } diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index b7c4f49987a..a9e3f5e2973 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -192,9 +192,8 @@ impl Import { for frame in frames { if frame.keyframe { // HDR metadata first, so a keyframe that also configures publishes once. - let hdr = crate::codec::color::access_unit(&frame.payload, self.length_size, 1, |h| { - h & 0x1f == NAL_TYPE_SEI - }); + let hdr = + crate::codec::color::access_unit(&frame.payload, self.length_size, 1, |h| h & 0x1f == NAL_TYPE_SEI); self.catalog.hdr(&mut self.track, hdr)?; // avc1 config arrives out-of-band via initialize(); avc3 carries the SPS inline. if self.length_size.is_none() @@ -329,13 +328,19 @@ mod tests { frames.extend(split.flush(pts).expect("flush keyframe")); import.decode(frames).expect("decode keyframe"); - let color = catalog.snapshot().video.renditions["video"].color.clone().expect("colour"); + let color = catalog.snapshot().video.renditions["video"] + .color + .clone() + .expect("colour"); assert_eq!( (color.primaries, color.transfer, color.matrix, color.full_range), (Some(9), Some(16), Some(9), Some(false)) ); let hdr = crate::codec::color::test::hdr10(); - assert_eq!((color.mastering, color.content_light), (hdr.mastering, hdr.content_light)); + assert_eq!( + (color.mastering, color.content_light), + (hdr.mastering, hdr.content_light) + ); } /// The stream outranks the caller's hint, which only fills what the stream leaves unsaid. @@ -346,15 +351,21 @@ mod tests { let mut hinted = hang::catalog::Color::default(); hinted.primaries = Some(9); hinted.content_light = crate::codec::color::test::hdr10().content_light; - let mut hint = crate::catalog::VideoHint::default(); - hint.color = Some(hinted); + let hint = crate::catalog::VideoHint { + color: Some(hinted), + ..Default::default() + }; let (track, catalog) = setup("video"); let mut import = Import::new(track, catalog.reserve(), hint).unwrap(); - let avcc = crate::codec::h264::build_avcc(&[Bytes::from_static(SPS_BT709)], &[Bytes::from_static(PPS)]).unwrap(); + let avcc = + crate::codec::h264::build_avcc(&[Bytes::from_static(SPS_BT709)], &[Bytes::from_static(PPS)]).unwrap(); import.initialize(&avcc).unwrap(); - let color = catalog.snapshot().video.renditions["video"].color.clone().expect("colour"); + let color = catalog.snapshot().video.renditions["video"] + .color + .clone() + .expect("colour"); assert_eq!(color.primaries, Some(1), "the SPS outranks the hint"); assert_eq!(color.content_light, crate::codec::color::test::hdr10().content_light); } diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index db342fc5bb1..d27facc9956 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -786,9 +786,7 @@ pub(crate) mod fixtures { 0x42, 0x00, 0x98, 0x96, 0x80, 0x00, 0x00, 0x03, 0x00, 0x32, 0x80, ]; /// x264's `cll=1000,400` SEI. - pub(crate) const SEI_CONTENT_LIGHT: &[u8] = &[ - 0x06, 0x90, 0x04, 0x03, 0xe8, 0x01, 0x90, 0x80, - ]; + pub(crate) const SEI_CONTENT_LIGHT: &[u8] = &[0x06, 0x90, 0x04, 0x03, 0xe8, 0x01, 0x90, 0x80]; } #[cfg(test)] diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index 20919dc52c1..442a16773d9 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -398,13 +398,19 @@ mod tests { frames.extend(split.flush(pts).expect("flush keyframe")); import.decode(frames).expect("decode keyframe"); - let color = catalog.snapshot().video.renditions["video"].color.clone().expect("colour"); + let color = catalog.snapshot().video.renditions["video"] + .color + .clone() + .expect("colour"); assert_eq!( (color.primaries, color.transfer, color.matrix, color.full_range), (Some(9), Some(16), Some(9), Some(false)) ); let hdr = crate::codec::color::test::hdr10(); - assert_eq!((color.mastering, color.content_light), (hdr.mastering, hdr.content_light)); + assert_eq!( + (color.mastering, color.content_light), + (hdr.mastering, hdr.content_light) + ); } fn setup(name: &str) -> (moq_net::track::Producer, crate::catalog::Producer) { diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index 88a6741751f..b6ee087e629 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -577,9 +577,7 @@ pub(crate) mod fixtures { 0x03, 0x00, 0x08, 0x00, 0x00, 0x03, 0x00, 0xc8, 0x40, ]; /// The same encode's `max-cll=1000,400` prefix SEI. - pub(crate) const SEI_CONTENT_LIGHT: &[u8] = &[ - 0x4e, 0x01, 0x90, 0x04, 0x03, 0xe8, 0x01, 0x90, 0x80, - ]; + pub(crate) const SEI_CONTENT_LIGHT: &[u8] = &[0x4e, 0x01, 0x90, 0x04, 0x03, 0xe8, 0x01, 0x90, 0x80]; /// The same encode's `master-display` prefix SEI: a 1000-nit BT.2020 display with a D65 white point. pub(crate) const SEI_MASTERING: &[u8] = &[ 0x4e, 0x01, 0x89, 0x18, 0x21, 0x34, 0x9b, 0xaa, 0x19, 0x96, 0x08, 0xfc, 0x8a, 0x48, 0x39, 0x08, 0x3d, 0x13, diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 225c0fa514c..bc957f2a742 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -2267,7 +2267,10 @@ fn color(primaries: u8, transfer: u8, matrix: u8, full_range: bool) -> hang::cat /// A colr box the SPS disagrees with loses to the SPS: the encoder signalled it. #[test] fn sps_color_outranks_colr() { - let catalog = run_fmp4(&avc1_init(crate::codec::h264::fixtures::SPS_BT709, Some(color(9, 16, 9, true)))); + let catalog = run_fmp4(&avc1_init( + crate::codec::h264::fixtures::SPS_BT709, + Some(color(9, 16, 9, true)), + )); let video = catalog.video.renditions.values().next().unwrap(); assert_eq!(video.color, Some(color(1, 1, 1, false))); } @@ -2275,7 +2278,10 @@ fn sps_color_outranks_colr() { /// A colr box fills what the SPS leaves unspecified. #[test] fn colr_fills_what_the_sps_leaves_unsaid() { - let catalog = run_fmp4(&avc1_init(crate::codec::h264::fixtures::SPS_IPB, Some(color(9, 16, 9, true)))); + let catalog = run_fmp4(&avc1_init( + crate::codec::h264::fixtures::SPS_IPB, + Some(color(9, 16, 9, true)), + )); let video = catalog.video.renditions.values().next().unwrap(); assert_eq!(video.color, Some(color(9, 16, 9, true))); } diff --git a/rs/moq-mux/src/container/mkv/export.rs b/rs/moq-mux/src/container/mkv/export.rs index 94b7fb514db..96636e272f1 100644 --- a/rs/moq-mux/src/container/mkv/export.rs +++ b/rs/moq-mux/src/container/mkv/export.rs @@ -761,7 +761,7 @@ pub(super) fn matroska_colour(color: &hang::catalog::Color) -> Option Result { diff --git a/rs/moq-mux/src/container/mkv/import.rs b/rs/moq-mux/src/container/mkv/import.rs index c17d445927b..8ddc00565ff 100644 --- a/rs/moq-mux/src/container/mkv/import.rs +++ b/rs/moq-mux/src/container/mkv/import.rs @@ -785,7 +785,6 @@ pub(super) fn matroska_color(children: &[MatroskaSpec]) -> Option, diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index 33aab79db5b..7be44474a1b 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -1119,7 +1119,11 @@ fn sps_color_outranks_colour_which_fills_its_gaps() { container.content_light = hdr.content_light; let colour = super::export::matroska_colour(&container).unwrap(); - let avcc = crate::codec::h264::build_avcc(&[bytes::Bytes::from_static(SPS_BT709)], &[bytes::Bytes::from_static(PPS)]).unwrap(); + let avcc = crate::codec::h264::build_avcc( + &[bytes::Bytes::from_static(SPS_BT709)], + &[bytes::Bytes::from_static(PPS)], + ) + .unwrap(); let entry = MatroskaSpec::TrackEntry(Master::Full(vec![ MatroskaSpec::TrackNumber(1), MatroskaSpec::TrackUID(1), @@ -1146,7 +1150,10 @@ fn sps_color_outranks_colour_which_fills_its_gaps() { (color.primaries, color.transfer, color.matrix, color.full_range), (Some(1), Some(1), Some(1), Some(false)) ); - assert_eq!((color.mastering, color.content_light), (hdr.mastering, hdr.content_light)); + assert_eq!( + (color.mastering, color.content_light), + (hdr.mastering, hdr.content_light) + ); } /// Every catalog colour field survives a Matroska Colour element. diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index f2e6664d3f4..5a666876ea6 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -42,6 +42,7 @@ fn rendition_hint(rendition: hang::catalog::VideoConfig) -> moq_mux::catalog::Vi hint.coded_height = rendition.coded_height; hint.display_aspect_width = rendition.display_aspect_width; hint.display_aspect_height = rendition.display_aspect_height; + hint.color = rendition.color; hint.framerate = rendition.framerate; hint.bitrate = rendition.bitrate; hint.optimize_for_latency = rendition.optimize_for_latency; From 0630520058564435f198e2c6f422c24256abd4e9 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 21:14:13 -0700 Subject: [PATCH 06/14] refactor(hang): keep Color's API to its fields Co-Authored-By: Claude Opus 5.5 --- rs/hang/src/catalog/video/color.rs | 9 --------- rs/moq-mux/src/codec/color.rs | 2 +- rs/moq-mux/src/codec/h265/sps.rs | 2 -- rs/moq-mux/src/container/mkv/import.rs | 4 ++-- 4 files changed, 3 insertions(+), 14 deletions(-) diff --git a/rs/hang/src/catalog/video/color.rs b/rs/hang/src/catalog/video/color.rs index 9bc61fe197d..4bcfe9461bf 100644 --- a/rs/hang/src/catalog/video/color.rs +++ b/rs/hang/src/catalog/video/color.rs @@ -31,13 +31,6 @@ pub struct Color { pub content_light: Option, } -impl Color { - /// True when no field is set, so the colour can be omitted. - pub fn is_empty(&self) -> bool { - self == &Self::default() - } -} - /// The colour volume of a mastering display, as SMPTE ST 2086. #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq)] #[serde(rename_all = "camelCase")] @@ -130,7 +123,5 @@ mod test { serde_json::to_value(&color).unwrap(), serde_json::json!({ "matrix": 1 }) ); - assert!(!color.is_empty()); - assert!(Color::default().is_empty()); } } diff --git a/rs/moq-mux/src/codec/color.rs b/rs/moq-mux/src/codec/color.rs index e6c2f8af220..603e50a114f 100644 --- a/rs/moq-mux/src/codec/color.rs +++ b/rs/moq-mux/src/codec/color.rs @@ -21,7 +21,7 @@ pub(crate) fn cicp(primaries: u8, transfer: u8, matrix: u8, full_range: Option Err(Error::SpsParse), } } -} -impl Sps { /// The colour the VUI's video signal type carries, when scuffle_h265 accepted the SPS. pub(crate) fn color(&self) -> Option { let signal = &self.full.as_ref()?.vui_parameters.as_ref()?.video_signal_type; diff --git a/rs/moq-mux/src/container/mkv/import.rs b/rs/moq-mux/src/container/mkv/import.rs index 8ddc00565ff..63a099d44bb 100644 --- a/rs/moq-mux/src/container/mkv/import.rs +++ b/rs/moq-mux/src/container/mkv/import.rs @@ -723,7 +723,7 @@ fn build_video_config( /// The colour a Matroska `Colour` element signals. Its code points are H.273's, `Range` counts 0 /// as unspecified, 1 as limited, and 2 as full, and a partial mastering display is dropped. pub(super) fn matroska_color(children: &[MatroskaSpec]) -> Option { - use hang::catalog::{Chromaticity, ContentLight, Mastering}; + use hang::catalog::{Chromaticity, Color, ContentLight, Mastering}; let (mut primaries, mut transfer, mut matrix, mut full_range) = (2, 2, 2, None); let (mut max_cll, mut max_fall) = (None, None); @@ -782,7 +782,7 @@ pub(super) fn matroska_color(children: &[MatroskaSpec]) -> Option Date: Fri, 16 Oct 2026 21:14:50 -0700 Subject: [PATCH 07/14] quest(m2): finish the catalog colour model Co-Authored-By: Claude Opus 5.5 --- quest/m2/README.md | 1 - quest/m2/color-catalog.md | 58 -------------------------------------- quest/m2/display-aspect.md | 4 --- quest/m2/webgpu-hdr.md | 6 +--- 4 files changed, 1 insertion(+), 68 deletions(-) delete mode 100644 quest/m2/color-catalog.md diff --git a/quest/m2/README.md b/quest/m2/README.md index 8acc5a2e83d..f2fe26796a6 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -82,7 +82,6 @@ ranked just below deterministic TS legs, which it requires. - [Go per-platform lib modules](/quest/m2/go-ffi-platform-modules.md) - a Go build downloads only the host's staticlib, and `go mod vendor` works - [Dart on iOS](/quest/m2/dart-ios.md) - prove the shipped iOS native asset actually loads on a device, which no CI can - [Mobile completion](/quest/m2/mobile-completion.md) - verify the Swift and Kotlin bindings over Rust-owned capture before closing #700 -- [Catalog colour](/quest/m2/color-catalog.md) - the catalog describes a rendition's colour and HDR properties, which the WebGPU HDR renderer reads - [Display aspect](/quest/m2/display-aspect.md) - H.264 and H.265 (Annex-B and hvcC) importers report a non-square sample aspect as the catalog display aspect, and AV1 too if it carries one - [WebGPU HDR](/quest/m2/webgpu-hdr.md) - HDR renditions play as HDR where the browser and display can show it, and tone-map to SDR elsewhere - [Grant stats label](/quest/m2/grant-label.md) - an auth grant's optional `label` names what its session's presence and live bytes count under in stats, apart from the path scope it may access diff --git a/quest/m2/color-catalog.md b/quest/m2/color-catalog.md deleted file mode 100644 index 83ff702a5c8..00000000000 --- a/quest/m2/color-catalog.md +++ /dev/null @@ -1,58 +0,0 @@ -# [M] Catalog colour model - -## Goal - -The Hang catalog describes a video rendition's colour: primaries, transfer -characteristics, matrix coefficients, range, and the HDR10 mastering display and -content light levels. A renderer can set up its pipeline from the catalog -instead of guessing or reparsing the bitstream. - -## Plan - -Deferred to m2 with HDR. The -[WebGPU HDR](/quest/m2/webgpu-hdr.md) renderer is the consumer this model -waited for. Encoder colour landed in m1 (#5808): moq-video's `Color` -carries primaries and transfer, so a catalog field can reuse its names. - -`rs/hang/src/catalog/video/mod.rs` has carried a bare `// TODO color space` since -the config was written. Two codecs already expose colour per their own syntax: -`VP9` carries primaries, transfer characteristics, matrix coefficients, and -range inside its codec string, and `AV1` has the same fields, which -`rs/moq-mux/src/codec/av1/import.rs` fills from the sequence header. They only -round-trip through muxers (the fMP4 exporter writes VP9's into `vpcC`); no -renderer consumes either, H.264 and H.265 have no equivalent, and there is no HDR10, -mastering display, or content light handling in `rs/` or `js/` at all. So an -HDR broadcast is delivered today and rendered as if it were SDR, whatever the -source signalled. - -Model the codec-neutral properties rather than one codec's syntax, so H.264 and -H.265 VUI, AV1 colour config, and a container's `colr` box all populate the same -fields. Generalize the existing VP9 and AV1 fields into that model rather than -adding a second colour source beside them: the codec-specific fields keep their -wire meaning (the VP9 codec string is defined by its spec) and feed the neutral -fields at import, so a renderer reads one place. Fill them at import from -whichever source the rendition provides, and emit them on export. - -Settle the precedence before adding the fields, because these sources disagree -in real files and a renderer must not see a different answer per import path. -Prefer the bitstream over the container: VUI and the AV1 colour config are what -the encoder actually signalled and travel with the elementary stream, while a -`colr` box is added by a muxer that may be wrong or stale. Record which source -won so a conflict is diagnosable rather than invisible, and treat an -unspecified value as absent rather than as a signalled default, so a partial -container box does not override a complete bitstream. - -Static display properties belong here rather than in a timed track. Mastering -display and content light level stay in the H.26x bitstream; the separation -study measured those messages and left them inline. This catalog can copy the -static values for a renderer when one exists. It does not move them out of the -access unit. - -Test each source of truth in isolation, a source that signals nothing, a -conflict between VUI and container resolving to the bitstream, a container box -that fills a gap the bitstream left unspecified, and an SDR round trip that -stays byte-identical. - -## Related - -- [WebGPU HDR](/quest/m2/webgpu-hdr.md) - the renderer that consumes this model diff --git a/quest/m2/display-aspect.md b/quest/m2/display-aspect.md index c7bbbfd8d2b..0ee1acb7e46 100644 --- a/quest/m2/display-aspect.md +++ b/quest/m2/display-aspect.md @@ -38,7 +38,3 @@ reader crops with checked math. Public API: doc only. Wire: the catalog's existing display aspect fields are populated in more cases. - -## Related - -- [Catalog colour](/quest/m2/color-catalog.md) - parses the same H.264/H.265 VUI for colour diff --git a/quest/m2/webgpu-hdr.md b/quest/m2/webgpu-hdr.md index c858e5ebfdd..938986b7ea7 100644 --- a/quest/m2/webgpu-hdr.md +++ b/quest/m2/webgpu-hdr.md @@ -29,7 +29,7 @@ Facts (2026-10): has less headroom. - `VideoFrame.colorSpace` gives primaries, transfer, matrix, and range (Chrome 94, Firefox 130, Safari 16.4). Mastering display and content light - levels come from [Catalog colour](/quest/m2/color-catalog.md). + levels come from the rendition's catalog `color`. Settle while building: whether the HDR path gives up zero copy (planes via `copyTo` and a custom conversion shader), and how the renderer detects @@ -40,7 +40,3 @@ Tests: an HDR10 rendition renders extended on Chromium desktop and on Safari where extended output is unavailable, and an SDR rendition is unchanged. Public API: none beyond the renderer. Wire: none. - -## Required - -- [Catalog colour](/quest/m2/color-catalog.md) - the mastering display and content light levels From 8232b528267637625ab2627095ba1b904585bc4f Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:21:42 -0700 Subject: [PATCH 08/14] fix(moq-mux): read colour from av1C configOBUs and VP8 vpcC The AV1 track importer now takes colour and HDR metadata OBUs from an av1C's configOBUs, and every av1C path keeps those metadata OBUs. VP8's vpcC is read like VP9's, and the exporter writes the catalog colour into it instead of mp4-atom's BT.709 placeholder. Co-Authored-By: Claude Opus 5.5 --- doc/concept/hang.md | 2 +- rs/moq-mux/src/codec/av1/import.rs | 71 ++++++++++++++++++-- rs/moq-mux/src/codec/color.rs | 6 +- rs/moq-mux/src/codec/video.rs | 2 +- rs/moq-mux/src/codec/vp8/mod.rs | 9 ++- rs/moq-mux/src/container/fmp4/import.rs | 8 +++ rs/moq-mux/src/container/fmp4/import_test.rs | 30 +++++++++ rs/moq-mux/src/container/fmp4/mod.rs | 2 +- 8 files changed, 118 insertions(+), 12 deletions(-) diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 618d872b683..e1ef5cda35a 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -67,7 +67,7 @@ A few things the catalog can express beyond decoder config: - **Track names.** The map key is a rendition's ID and, by default, its track name. A rendition sets `track` only when the two must differ, such as two renditions of same-named tracks in different broadcasts, or a rendition whose media changed and so needs a new name; readers subscribe to `track` when present. An ID can't name both a video and an audio rendition. - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. -- **Colour.** A video rendition may carry `color`: the H.273 primaries, transfer, matrix, and range, plus the HDR10 mastering display and content light levels, so a renderer can set up HDR without parsing the bitstream. The Rust importers fill it from the SPS VUI, AV1 sequence header, and VP9 header, then fill gaps from a container's `colr` box or Matroska `Colour` element; the stream wins a disagreement. The HDR10 values come from H.264 and H.265 SEI or AV1 metadata OBUs in MPEG-TS and raw elementary streams, or from Matroska, but not yet from fMP4 or FLV. Nothing in `@moq/watch` reads it yet. +- **Colour.** A video rendition may carry `color`: the H.273 primaries, transfer, matrix, and range, plus the HDR10 mastering display and content light levels, so a renderer can set up HDR without parsing the bitstream. The Rust importers fill it from the SPS VUI, AV1 sequence header, and VP9 header, then fill gaps from a container's `colr` box or Matroska `Colour` element; the stream wins a disagreement. The HDR10 values come from H.264 and H.265 SEI or AV1 metadata OBUs in MPEG-TS and raw elementary streams, an AV1 track's av1C, or Matroska, but not yet from H.264 or H.265 in fMP4 or FLV. Nothing in `@moq/watch` reads it yet. - **Disabled renditions.** An audio or video rendition with `enabled: false` has no frames coming, and a player must not select it. Pausing a rendition in `@moq/publish` keeps it in the catalog this way rather than removing it, so a mute is a one-field update; a disabled video rendition encodes one black keyframe first, so a player that predates the field shows black instead of a frozen picture. The field is written only when false. The older video `stalled` field is no longer written, and readers ignore it. - **Preference.** A video rendition may carry a signed `preference`, default `0`. A player keeps only the highest preference it can decode, then picks by size and bitrate within it, as do the single-rendition egresses (RTMP play, WHEP, FLV, the transcoder's source). A publisher that adds its own H.264 copy of an H.265 source marks the copy `-1`, so only a viewer that can't decode H.265 pulls it. Picking a rendition by name still overrides it. The field is written only when nonzero. - **Archive.** A broadcast may advertise an `archive` entry mapping each track, the catalog included, to its own timeline track (a small index of the track's spans: timing plus group and frame positions) and fixed nominal record duration, plus the replay MoQ path, object-store URL, and format version when recorded. Media records end at the first group boundary at or beyond a 4s target; sparse timelines such as the catalog declare zero. Tracks commit and expire independently. An untimed track has no timing to index, so its timeline declares no duration and holds one record per group: it can be recorded and replayed, but not turned into HLS or kept in a DVR. The timelines are what let the [HLS gateway](/bin/hls) build playlists without subscribing to media. A reader that cannot decode the `archive` entry, such as one from a different version, ignores it and loses the timelines and replay, but live media still plays. diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index daae5fab7bd..8e65c99f89f 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -94,7 +94,7 @@ impl Import { let twelve_bit = ((data[2] >> 5) & 0x01) == 1; // Resolution is unknown from av1C; it's filled when the first sequence header arrives. - let config = hang::catalog::VideoConfig::new(hang::catalog::AV1 { + let mut config = hang::catalog::VideoConfig::new(hang::catalog::AV1 { profile: seq_profile, level: seq_level_idx, tier: if tier { 'H' } else { 'M' }, @@ -108,6 +108,8 @@ impl Import { matrix_coefficients: 1, full_range: false, }); + // configOBUs follow the four header bytes; samples need not repeat what they carry. + config.color = color(&data[4..]); self.apply_config(config)?; Ok(()) } @@ -291,10 +293,14 @@ fn sequence_color(seq_header: &SequenceHeaderObu) -> Option Option { - let sequence = parse_sequence_header(&find_sequence_header(obus)?).ok()??; - sequence_color(&sequence) + let mut color = find_sequence_header(obus) + .and_then(|seq| parse_sequence_header(&seq).ok().flatten()) + .and_then(|seq| sequence_color(&seq)); + find_hdr(&Bytes::copy_from_slice(obus)).apply(&mut color); + color } /// Read encoded dimensions from the first parseable sequence header in a frame. @@ -344,3 +350,60 @@ fn find_sequence_header(payload: &[u8]) -> Option { } obus.flush().ok().flatten().filter(|obu| is_sequence_header(obu)) } + +#[cfg(test)] +mod tests { + use super::*; + + /// A metadata OBU (type 5, with a size field) carrying `payload`. + fn metadata_obu(payload: &[u8]) -> Vec { + let mut obu = vec![(5 << 3) | 0x02, payload.len() as u8]; + obu.extend_from_slice(payload); + obu + } + + /// HDR10 metadata OBUs: a 1000-nit BT.2020 mastering display, then MaxCLL 1000 and MaxFALL 400. + fn hdr10_obus() -> Vec { + let mut mdcv = vec![2]; + for v in [46_399u16, 19_136, 11_141, 52_232, 8_585, 3_014, 20_493, 21_561] { + mdcv.extend_from_slice(&v.to_be_bytes()); + } + mdcv.extend_from_slice(&(1000u32 << 8).to_be_bytes()); + mdcv.extend_from_slice(&82u32.to_be_bytes()); + let mut obus = metadata_obu(&mdcv); + obus.extend(metadata_obu(&[1, 0x03, 0xE8, 0x01, 0x90])); + obus + } + + /// An av1C's configOBUs may carry the static HDR metadata alone; it is not dropped. + #[test] + fn config_obus_carry_hdr_metadata() { + let color = color(&hdr10_obus()).expect("HDR metadata"); + assert_eq!(color.mastering.map(|m| m.max_luminance), Some(1000.0)); + assert_eq!(color.content_light, crate::codec::color::test::hdr10().content_light); + assert_eq!(color.primaries, None); + } + + /// A track initialized from an av1C publishes the colour its configOBUs carry, since the + /// samples need not repeat it. + #[tokio::test(start_paused = true)] + async fn av1c_initialize_publishes_config_obu_color() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let track = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let mut import = Import::new(track, catalog.reserve(), Default::default()).unwrap(); + + let mut av1c = vec![0x81, 0x08, 0x0c, 0x00]; + av1c.extend(hdr10_obus()); + import.initialize(&av1c).unwrap(); + + let color = catalog.snapshot().video.renditions["video"] + .color + .clone() + .expect("colour"); + assert_eq!(color.content_light, crate::codec::color::test::hdr10().content_light); + assert!(color.mastering.is_some()); + } +} diff --git a/rs/moq-mux/src/codec/color.rs b/rs/moq-mux/src/codec/color.rs index 603e50a114f..10da0a141b1 100644 --- a/rs/moq-mux/src/codec/color.rs +++ b/rs/moq-mux/src/codec/color.rs @@ -80,12 +80,12 @@ impl Hdr { self.content_light = other.content_light.or(self.content_light); } - /// Set these fields on `config`'s colour; they come from the bitstream, so they replace a hint. - pub(crate) fn apply(&self, config: &mut hang::catalog::VideoConfig) { + /// Set these fields on `color`; they come from the bitstream, so they replace a hint. + pub(crate) fn apply(&self, color: &mut Option) { if *self == Self::default() { return; } - let color = config.color.get_or_insert_with(Color::default); + let color = color.get_or_insert_with(Color::default); color.mastering = self.mastering.or(color.mastering); color.content_light = self.content_light.or(color.content_light); } diff --git a/rs/moq-mux/src/codec/video.rs b/rs/moq-mux/src/codec/video.rs index e7cbf90ea68..eca69815ec5 100644 --- a/rs/moq-mux/src/codec/video.rs +++ b/rs/moq-mux/src/codec/video.rs @@ -92,7 +92,7 @@ impl Catalog { /// reject a reconfiguration. pub(crate) fn publish(&mut self, track: &mut Track, mut config: hang::catalog::VideoConfig) -> crate::Result<()> { self.resolved = Some(config.clone()); - self.hdr.apply(&mut config); + self.hdr.apply(&mut config.color); self.hint.apply(&mut config); if self.last.as_ref() == Some(&config) { return Ok(()); diff --git a/rs/moq-mux/src/codec/vp8/mod.rs b/rs/moq-mux/src/codec/vp8/mod.rs index da9559aedcd..815a355e777 100644 --- a/rs/moq-mux/src/codec/vp8/mod.rs +++ b/rs/moq-mux/src/codec/vp8/mod.rs @@ -85,12 +85,17 @@ impl FrameHeader { /// /// VP8 is always 8-bit 4:2:0 with no out-of-band parameters, so the box carries /// fixed placeholders. Profile 0 / level 0 are the standard "unspecified" -/// values. See https://www.webmproject.org/vp9/mp4/. -pub(crate) fn vpcc() -> mp4_atom::VpcC { +/// values. See https://www.webmproject.org/vp9/mp4/. The colour fields carry the +/// catalog's, unspecified (2) when unknown rather than mp4-atom's BT.709 default. +pub(crate) fn vpcc(color: Option<&hang::catalog::Color>) -> mp4_atom::VpcC { mp4_atom::VpcC { profile: 0, level: 0, bit_depth: 8, + color_primaries: color.and_then(|c| c.primaries).unwrap_or(2), + transfer_characteristics: color.and_then(|c| c.transfer).unwrap_or(2), + matrix_coefficients: color.and_then(|c| c.matrix).unwrap_or(2), + video_full_range_flag: color.and_then(|c| c.full_range).unwrap_or(false), ..Default::default() } } diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index 35a404594ad..2746775289f 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -481,6 +481,14 @@ impl Import { ), mp4_atom::Codec::Vp08(vp08) => { let mut config = VideoConfig::new(VideoCodec::VP8); + // vpcC carries VP8's colour too; its range is always explicit. + let vpcc = &vp08.vpcc; + config.color = crate::codec::color::cicp( + vpcc.color_primaries, + vpcc.transfer_characteristics, + vpcc.matrix_coefficients, + Some(vpcc.video_full_range_flag), + ); config.coded_width = Some(vp08.visual.width as _); config.coded_height = Some(vp08.visual.height as _); config.container = container; diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index bc957f2a742..796e1c03674 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -2318,3 +2318,33 @@ fn sdr_color_round_trips_byte_identical() { }; assert_eq!(super::encode_init(None, vec![trak], vec![trex]).unwrap(), init); } + +/// VP8's vpcC carries its colour as VP9's does: export writes the catalog's, unspecified when +/// unknown rather than mp4-atom's BT.709 default, and import reads it back. +#[test] +fn vp8_color_round_trips_through_vpcc() { + use hang::catalog::{VideoCodec, VideoConfig}; + + let init = |color: Option| { + let mut config = VideoConfig::new(VideoCodec::VP8); + config.coded_width = Some(64); + config.coded_height = Some(64); + config.color = color; + let trak = super::synthesize_video_trak(1, 90_000, &config, None).unwrap(); + let trex = mp4_atom::Trex { + track_id: 1, + default_sample_description_index: 1, + ..Default::default() + }; + super::encode_init(None, vec![trak], vec![trex]).unwrap() + }; + + let catalog = run_fmp4(&init(Some(color(9, 18, 9, true)))); + let video = catalog.video.renditions.values().next().unwrap(); + assert_eq!(video.color, Some(color(9, 18, 9, true))); + + // Without a catalog colour, only vpcC's always-explicit range survives, never a made-up BT.709. + let catalog = run_fmp4(&init(None)); + let color = catalog.video.renditions.values().next().unwrap().color.clone().unwrap(); + assert_eq!((color.primaries, color.transfer, color.matrix), (None, None, None)); +} diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index 3a616fffebe..e5028bc7c26 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -870,7 +870,7 @@ pub(crate) fn synthesize_video_trak( }), VideoCodec::VP8 => mp4_atom::Codec::from(mp4_atom::Vp08 { visual, - vpcc: crate::codec::vp8::vpcc(), + vpcc: crate::codec::vp8::vpcc(config.color.as_ref()), colr, ..Default::default() }), From ed9620c1ea6f974887ff5234154fe9146dded2a9 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:07:34 -0700 Subject: [PATCH 09/14] fix(moq-mux): reset HDR metadata per coded sequence, refuse unrepresentable colour A new SPS or sequence header replaces the cached HDR metadata instead of merging, so an HDR-to-SDR switch drops stale HDR10 values. vpcC now carries the resolved colour so it can't contradict colr. Matroska Colour and colr values with no catalog representation are refused rather than clamped. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/codec/av1/import.rs | 9 ++-- rs/moq-mux/src/codec/h264/import.rs | 46 +++++++++++++++++--- rs/moq-mux/src/codec/h265/import.rs | 11 ++--- rs/moq-mux/src/codec/video.rs | 15 ++++++- rs/moq-mux/src/codec/vp8/mod.rs | 13 +++--- rs/moq-mux/src/container/fmp4/import.rs | 3 +- rs/moq-mux/src/container/fmp4/import_test.rs | 30 +++++++++++++ rs/moq-mux/src/container/fmp4/mod.rs | 41 ++++++++++++----- rs/moq-mux/src/container/mkv/import.rs | 35 ++++++++------- rs/moq-mux/src/container/mkv/import_test.rs | 14 +++++- rs/moq-mux/src/container/mkv/mod.rs | 4 ++ 11 files changed, 171 insertions(+), 50 deletions(-) diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index 8e65c99f89f..02e02a3fcdb 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -243,9 +243,12 @@ impl Import { fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { for frame in frames { if frame.keyframe { - // HDR metadata first, so a keyframe that also configures publishes once. - self.catalog.hdr(&mut self.track, find_hdr(&frame.payload))?; - if let Some(seq) = find_sequence_header(&frame.payload) { + let seq = find_sequence_header(&frame.payload); + // A new sequence header starts a coded sequence whose HDR metadata replaces the last + // one's. It goes first, so a keyframe that also configures publishes once. + let sequence = seq.as_ref().is_some_and(|seq| self.last_seq.as_ref() != Some(seq)); + self.catalog.hdr(&mut self.track, find_hdr(&frame.payload), sequence)?; + if let Some(seq) = seq { self.configure_from_seq(&seq)?; } } diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index a9e3f5e2973..3bed77f2de0 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -191,14 +191,15 @@ impl Import { fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { for frame in frames { if frame.keyframe { - // HDR metadata first, so a keyframe that also configures publishes once. let hdr = crate::codec::color::access_unit(&frame.payload, self.length_size, 1, |h| h & 0x1f == NAL_TYPE_SEI); - self.catalog.hdr(&mut self.track, hdr)?; // avc1 config arrives out-of-band via initialize(); avc3 carries the SPS inline. - if self.length_size.is_none() - && let Some(sps) = find_sps(&frame.payload) - { + let sps = self.length_size.is_none().then(|| find_sps(&frame.payload)).flatten(); + // A new SPS starts a coded sequence whose HDR metadata replaces the last one's. It goes + // first, so a keyframe that also configures publishes once. + let sequence = sps.as_ref().is_some_and(|sps| self.last_sps.as_ref() != Some(sps)); + self.catalog.hdr(&mut self.track, hdr, sequence)?; + if let Some(sps) = sps { self.configure_from_sps(&sps)?; } } @@ -343,6 +344,41 @@ mod tests { ); } + /// HDR metadata merges across keyframes of one coded sequence, and a new SPS drops it, so an + /// HDR-to-SDR switch never keeps stale HDR10 values. + #[tokio::test(start_paused = true)] + async fn a_new_sps_drops_stale_hdr_metadata() { + use crate::codec::h264::fixtures::{PPS, SEI_CONTENT_LIGHT, SEI_MASTERING, SPS_BT709, SPS_HDR10}; + + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.reserve(), Default::default()).unwrap(); + let mut split = Split::new(); + let mut keyframe = |ms: u64, nals: &[&[u8]]| { + let mut annexb = BytesMut::new(); + for nal in nals { + annexb.extend_from_slice(&[0, 0, 0, 1]); + annexb.extend_from_slice(nal); + } + let pts = moq_net::Timestamp::from_millis(ms).unwrap(); + let mut frames = split.decode(&annexb, pts).expect("split keyframe"); + frames.extend(split.flush(pts).expect("flush keyframe")); + import.decode(frames).expect("decode keyframe"); + catalog.snapshot().video.renditions["video"] + .color + .clone() + .expect("colour") + }; + + let color = keyframe(0, &[SPS_HDR10, PPS, SEI_MASTERING, SEI_CONTENT_LIGHT, idr]); + assert!(color.mastering.is_some()); + let color = keyframe(40, &[SPS_HDR10, PPS, idr]); + assert!(color.mastering.is_some(), "the same sequence keeps its metadata"); + let color = keyframe(80, &[SPS_BT709, PPS, idr]); + assert_eq!(color.primaries, Some(1)); + assert_eq!((color.mastering, color.content_light), (None, None)); + } + /// The stream outranks the caller's hint, which only fills what the stream leaves unsaid. #[tokio::test(start_paused = true)] async fn stream_color_outranks_the_hint() { diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index 442a16773d9..b570a27fcea 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -194,17 +194,18 @@ impl Import { fn write_frames(&mut self, frames: impl IntoIterator) -> Result<()> { for frame in frames { if frame.keyframe { - // HDR metadata first, so a keyframe that also configures publishes once. let hdr = crate::codec::color::access_unit(&frame.payload, self.length_size, 2, |h| { nal_unit_type(h) == scuffle_h265::NALUnitType::PrefixSeiNut }); - self.catalog.hdr(&mut self.track, hdr)?; // hvc1 config arrives out-of-band via initialize(); hev1 carries the SPS inline. // Scanning an hvc1 payload as Annex-B would misread a length prefix as a start // code (a 322-byte NAL prefixes as `00 00 01 42`, an SPS NAL header). - if self.length_size.is_none() - && let Some(sps) = find_sps(&frame.payload) - { + let sps = self.length_size.is_none().then(|| find_sps(&frame.payload)).flatten(); + // A new SPS starts a coded sequence whose HDR metadata replaces the last one's. It goes + // first, so a keyframe that also configures publishes once. + let sequence = sps.as_ref().is_some_and(|sps| self.last_sps.as_ref() != Some(sps)); + self.catalog.hdr(&mut self.track, hdr, sequence)?; + if let Some(sps) = sps { self.configure_from_sps(&sps)?; } } diff --git a/rs/moq-mux/src/codec/video.rs b/rs/moq-mux/src/codec/video.rs index eca69815ec5..29ff1652b57 100644 --- a/rs/moq-mux/src/codec/video.rs +++ b/rs/moq-mux/src/codec/video.rs @@ -104,7 +104,20 @@ impl Catalog { } /// Overlay the HDR metadata a keyframe carried in band, republishing the last config if it changed. - pub(crate) fn hdr(&mut self, track: &mut Track, hdr: crate::codec::color::Hdr) -> crate::Result<()> { + /// + /// `sequence` marks a keyframe that starts a new coded sequence (a new SPS or sequence header): + /// its metadata replaces the old rather than merging, so an HDR-to-SDR switch drops stale HDR10 + /// values, and it is left for the new config's publish. + pub(crate) fn hdr( + &mut self, + track: &mut Track, + hdr: crate::codec::color::Hdr, + sequence: bool, + ) -> crate::Result<()> { + if sequence { + self.hdr = hdr; + return Ok(()); + } let mut next = self.hdr; next.update(hdr); if next == self.hdr { diff --git a/rs/moq-mux/src/codec/vp8/mod.rs b/rs/moq-mux/src/codec/vp8/mod.rs index 815a355e777..d5cdb24a3fb 100644 --- a/rs/moq-mux/src/codec/vp8/mod.rs +++ b/rs/moq-mux/src/codec/vp8/mod.rs @@ -85,17 +85,16 @@ impl FrameHeader { /// /// VP8 is always 8-bit 4:2:0 with no out-of-band parameters, so the box carries /// fixed placeholders. Profile 0 / level 0 are the standard "unspecified" -/// values. See https://www.webmproject.org/vp9/mp4/. The colour fields carry the -/// catalog's, unspecified (2) when unknown rather than mp4-atom's BT.709 default. -pub(crate) fn vpcc(color: Option<&hang::catalog::Color>) -> mp4_atom::VpcC { +/// values. See https://www.webmproject.org/vp9/mp4/. The colour fields are +/// unspecified (2) rather than mp4-atom's BT.709 default. +pub(crate) fn vpcc() -> mp4_atom::VpcC { mp4_atom::VpcC { profile: 0, level: 0, bit_depth: 8, - color_primaries: color.and_then(|c| c.primaries).unwrap_or(2), - transfer_characteristics: color.and_then(|c| c.transfer).unwrap_or(2), - matrix_coefficients: color.and_then(|c| c.matrix).unwrap_or(2), - video_full_range_flag: color.and_then(|c| c.full_range).unwrap_or(false), + color_primaries: 2, + transfer_characteristics: 2, + matrix_coefficients: 2, ..Default::default() } } diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index 2746775289f..9786eab14ec 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -527,7 +527,8 @@ impl Import { unsupported => return Err(Error::UnsupportedCodec(Box::new(unsupported.clone())).into()), }; // The codec config outranks the colr box a muxer added, which only fills its gaps. - crate::codec::color::fill(&mut config.color, colr.and_then(super::colr_color).as_ref(), "colr"); + let colr = colr.map(super::colr_color).transpose()?.flatten(); + crate::codec::color::fill(&mut config.color, colr.as_ref(), "colr"); Ok(config) } diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 796e1c03674..d29437fe9c6 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -2348,3 +2348,33 @@ fn vp8_color_round_trips_through_vpcc() { let color = catalog.video.renditions.values().next().unwrap().color.clone().unwrap(); assert_eq!((color.primaries, color.transfer, color.matrix), (None, None, None)); } + +/// A VP9 codec string holding a container's placeholder BT.709 does not contradict the resolved +/// colour: export writes the catalog colour into vpcC too, so a reimport keeps it. +#[test] +fn vp9_vpcc_carries_the_resolved_color() { + use hang::catalog::{VP9, VideoConfig}; + + let mut config = VideoConfig::new(VP9 { + profile: 0, + level: 0, + bit_depth: 8, + chroma_subsampling: 1, + color_primaries: 1, + transfer_characteristics: 1, + matrix_coefficients: 1, + full_range: false, + }); + config.coded_width = Some(64); + config.coded_height = Some(64); + config.color = Some(color(9, 16, 9, false)); + let trak = super::synthesize_video_trak(1, 90_000, &config, None).unwrap(); + let trex = mp4_atom::Trex { + track_id: 1, + default_sample_description_index: 1, + ..Default::default() + }; + let catalog = run_fmp4(&super::encode_init(None, vec![trak], vec![trex]).unwrap()); + let video = catalog.video.renditions.values().next().unwrap(); + assert_eq!(video.color, Some(color(9, 16, 9, false))); +} diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index e5028bc7c26..90f5a5c4e54 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -113,6 +113,10 @@ pub enum Error { #[error("unknown codec: {0:?}")] UnknownCodec(mp4_atom::FourCC), + /// A `colr` box code point past H.273's 255. + #[error("invalid colr code point: {0}")] + InvalidColr(u16), + #[error("unsupported codec: {0:?}")] UnsupportedCodec(Box), @@ -745,8 +749,9 @@ fn trun_duration(duration: Timestamp, timescale: moq_net::Timescale) -> Result Option { +/// QuickTime's `nclc` has no range, and an ICC profile has no code points at all. A code point +/// past 255 has no H.273 meaning, so it is refused. +pub(crate) fn colr_color(colr: &mp4_atom::Colr) -> Result> { let mp4_atom::Colr::Nclx { colour_primaries, transfer_characteristics, @@ -754,16 +759,15 @@ pub(crate) fn colr_color(colr: &mp4_atom::Colr) -> Option full_range_flag, } = *colr else { - return None; + return Ok(None); }; - // A code point past 255 is reserved in H.273, so it reads as unspecified. - let code = |value: u16| u8::try_from(value).unwrap_or(2); - crate::codec::color::cicp( - code(colour_primaries), - code(transfer_characteristics), - code(matrix_coefficients), + let code = |value: u16| u8::try_from(value).map_err(|_| Error::InvalidColr(value)); + Ok(crate::codec::color::cicp( + code(colour_primaries)?, + code(transfer_characteristics)?, + code(matrix_coefficients)?, Some(full_range_flag), - ) + )) } /// The `nclx` box for a catalog colour, or `None` when it has no code point or range to carry. @@ -781,6 +785,19 @@ fn color_colr(color: Option<&hang::catalog::Color>) -> Option { }) } +/// Overlay the catalog colour on a `vpcC`, so it never contradicts the `colr` box written beside +/// it; a VP9 codec string may still carry a container's placeholder BT.709. +fn vpcc_color(mut vpcc: mp4_atom::VpcC, color: Option<&hang::catalog::Color>) -> mp4_atom::VpcC { + let Some(color) = color else { + return vpcc; + }; + vpcc.color_primaries = color.primaries.unwrap_or(vpcc.color_primaries); + vpcc.transfer_characteristics = color.transfer.unwrap_or(vpcc.transfer_characteristics); + vpcc.matrix_coefficients = color.matrix.unwrap_or(vpcc.matrix_coefficients); + vpcc.video_full_range_flag = color.full_range.unwrap_or(vpcc.video_full_range_flag); + vpcc +} + pub(crate) fn synthesize_video_trak( track_id: u32, timescale: u64, @@ -870,13 +887,13 @@ pub(crate) fn synthesize_video_trak( }), VideoCodec::VP8 => mp4_atom::Codec::from(mp4_atom::Vp08 { visual, - vpcc: crate::codec::vp8::vpcc(config.color.as_ref()), + vpcc: vpcc_color(crate::codec::vp8::vpcc(), config.color.as_ref()), colr, ..Default::default() }), VideoCodec::VP9(vp9) => mp4_atom::Codec::from(mp4_atom::Vp09 { visual, - vpcc: crate::codec::vp9::vpcc(vp9), + vpcc: vpcc_color(crate::codec::vp9::vpcc(vp9), config.color.as_ref()), colr, ..Default::default() }), diff --git a/rs/moq-mux/src/container/mkv/import.rs b/rs/moq-mux/src/container/mkv/import.rs index 63a099d44bb..fec670bd145 100644 --- a/rs/moq-mux/src/container/mkv/import.rs +++ b/rs/moq-mux/src/container/mkv/import.rs @@ -709,36 +709,41 @@ fn build_video_config( config.coded_height = height; } // The codec config outranks the Colour element a muxer added, which only fills its gaps. - let colour = video_children.and_then(|cs| { - cs.iter().find_map(|c| match c { - MatroskaSpec::Colour(Master::Full(colour)) => matroska_color(colour), - _ => None, + let colour = video_children + .and_then(|cs| { + cs.iter().find_map(|c| match c { + MatroskaSpec::Colour(Master::Full(colour)) => Some(colour), + _ => None, + }) }) - }); + .map(|colour| matroska_color(colour)) + .transpose()? + .flatten(); crate::codec::color::fill(&mut config.color, colour.as_ref(), "Colour"); Ok(config) } /// The colour a Matroska `Colour` element signals. Its code points are H.273's, `Range` counts 0 -/// as unspecified, 1 as limited, and 2 as full, and a partial mastering display is dropped. -pub(super) fn matroska_color(children: &[MatroskaSpec]) -> Option { +/// as unspecified, 1 as limited, and 2 as full, and a partial mastering display is dropped. A code +/// point past 255 or a light level past 65535 has no representation, so it is refused. +pub(super) fn matroska_color(children: &[MatroskaSpec]) -> Result> { use hang::catalog::{Chromaticity, Color, ContentLight, Mastering}; let (mut primaries, mut transfer, mut matrix, mut full_range) = (2, 2, 2, None); let (mut max_cll, mut max_fall) = (None, None); let mut mastering = None; - // A code point past 255 is reserved in H.273, so it reads as unspecified. - let code = |value: u64| u8::try_from(value).unwrap_or(2); + let code = |value: u64| u8::try_from(value).map_err(|_| Error::InvalidColour("code point")); + let light = |value: u64| u16::try_from(value).map_err(|_| Error::InvalidColour("light level")); for child in children { match child { - MatroskaSpec::Primaries(v) => primaries = code(*v), - MatroskaSpec::TransferCharacteristics(v) => transfer = code(*v), - MatroskaSpec::MatrixCoefficients(v) => matrix = code(*v), + MatroskaSpec::Primaries(v) => primaries = code(*v)?, + MatroskaSpec::TransferCharacteristics(v) => transfer = code(*v)?, + MatroskaSpec::MatrixCoefficients(v) => matrix = code(*v)?, MatroskaSpec::Range(1) => full_range = Some(false), MatroskaSpec::Range(2) => full_range = Some(true), - MatroskaSpec::MaxCLL(v) => max_cll = Some(u16::try_from(*v).unwrap_or(u16::MAX)), - MatroskaSpec::MaxFALL(v) => max_fall = Some(u16::try_from(*v).unwrap_or(u16::MAX)), + MatroskaSpec::MaxCLL(v) => max_cll = Some(light(*v)?), + MatroskaSpec::MaxFALL(v) => max_fall = Some(light(*v)?), MatroskaSpec::MasteringMetadata(Master::Full(fields)) => { // Red, green, blue, and white x and y, then the max and min luminance. let mut v = [None; 10]; @@ -782,7 +787,7 @@ pub(super) fn matroska_color(children: &[MatroskaSpec]) -> Option Date: Sat, 17 Oct 2026 00:35:49 -0700 Subject: [PATCH 10/14] fix(moq-mux): drop HDR metadata on a new config record, keep vpcC free of placeholders A new avcC, hvcC, or av1C starts a new coded sequence, so cached in-band HDR metadata is cleared. A partial catalog colour writes the missing vpcC fields as unspecified instead of a codec string's placeholder. @moq/hang exports Color and ColorSchema from its catalog entrypoint. Co-Authored-By: Claude Opus 5.5 --- js/hang/src/catalog/index.ts | 4 +-- js/hang/src/catalog/video.test.ts | 1 + rs/moq-mux/src/codec/av1/import.rs | 4 ++- rs/moq-mux/src/codec/h264/import.rs | 32 ++++++++++++++++++++ rs/moq-mux/src/codec/h265/import.rs | 2 ++ rs/moq-mux/src/container/fmp4/import_test.rs | 28 +++++++++++------ rs/moq-mux/src/container/fmp4/mod.rs | 13 ++++---- 7 files changed, 66 insertions(+), 18 deletions(-) diff --git a/js/hang/src/catalog/index.ts b/js/hang/src/catalog/index.ts index 27e1e00c5a1..f79292cd9f1 100644 --- a/js/hang/src/catalog/index.ts +++ b/js/hang/src/catalog/index.ts @@ -37,5 +37,5 @@ export * from "./root"; export * from "./stats"; export * from "./text"; export * from "./track"; -export type { VideoConfig } from "./video"; -export { VideoConfigSchema, VideoSchema } from "./video"; +export type { Color, VideoConfig } from "./video"; +export { ColorSchema, VideoConfigSchema, VideoSchema } from "./video"; diff --git a/js/hang/src/catalog/video.test.ts b/js/hang/src/catalog/video.test.ts index 0cc5899955b..447c4b68924 100644 --- a/js/hang/src/catalog/video.test.ts +++ b/js/hang/src/catalog/video.test.ts @@ -185,6 +185,7 @@ test("color parses the H.273 code points and HDR10 metadata a Rust publisher wri }; const config = VideoConfigSchema.parse({ codec: "hvc1.2.4.L93.B0", container: { kind: "legacy" }, color }); expect(config.color).toEqual(color as typeof config.color); + expect(Catalog.ColorSchema.parse(color)).toEqual(config.color as Catalog.Color); expect(VideoConfigSchema.parse({ codec: "vp8", container: { kind: "legacy" } }).color).toBeUndefined(); }); diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index 02e02a3fcdb..6f6518b32da 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -108,8 +108,10 @@ impl Import { matrix_coefficients: 1, full_range: false, }); - // configOBUs follow the four header bytes; samples need not repeat what they carry. + // configOBUs follow the four header bytes; samples need not repeat what they carry. A new + // record starts a new coded sequence, so earlier in-band HDR metadata no longer applies. config.color = color(&data[4..]); + self.catalog.hdr(&mut self.track, Default::default(), true)?; self.apply_config(config)?; Ok(()) } diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index 3bed77f2de0..446db44bfed 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -86,6 +86,8 @@ impl Import { // Only switch to avc1 mode once the avcC actually parses, so a parse failure leaves the // importer in avc3 mode where inline-SPS keyframes still self-initialize. let config = config_from_avcc(avcc_bytes)?; + // A new record starts a new coded sequence, whose HDR metadata the samples will carry. + self.catalog.hdr(&mut self.track, Default::default(), true)?; self.length_size = Some(super::Avcc::parse(avcc_bytes)?.length_size); self.apply_config(config)?; Ok(()) @@ -379,6 +381,36 @@ mod tests { assert_eq!((color.mastering, color.content_light), (None, None)); } + /// A new out-of-band avcC starts a new coded sequence, so in-band HDR metadata from the last one + /// is dropped. + #[tokio::test(start_paused = true)] + async fn a_new_avcc_drops_stale_hdr_metadata() { + use crate::codec::h264::fixtures::{PPS, SEI_CONTENT_LIGHT, SEI_MASTERING, SPS_BT709, SPS_HDR10}; + + let avcc = |sps: &'static [u8]| { + crate::codec::h264::build_avcc(&[Bytes::from_static(sps)], &[Bytes::from_static(PPS)]).unwrap() + }; + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.reserve(), Default::default()).unwrap(); + import.initialize(&avcc(SPS_HDR10)).unwrap(); + + let mut au = Vec::new(); + for nal in [SEI_MASTERING, SEI_CONTENT_LIGHT, &[0x65, 0x88, 0x84, 0x21][..]] { + au.extend_from_slice(&(nal.len() as u32).to_be_bytes()); + au.extend_from_slice(nal); + } + let pts = moq_net::Timestamp::from_micros(0).unwrap(); + import + .decode([crate::codec::h264::avc1_frame(au, 4, pts).unwrap()]) + .unwrap(); + let color = || catalog.snapshot().video.renditions["video"].color.clone().unwrap(); + assert!(color().mastering.is_some()); + + import.initialize(&avcc(SPS_BT709)).unwrap(); + assert_eq!(color().primaries, Some(1)); + assert_eq!((color().mastering, color().content_light), (None, None)); + } + /// The stream outranks the caller's hint, which only fills what the stream leaves unsaid. #[tokio::test(start_paused = true)] async fn stream_color_outranks_the_hint() { diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index b570a27fcea..b6036e5bde6 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -90,6 +90,8 @@ impl Import { // Only switch to hvc1 mode once the hvcC actually parses, so a parse failure leaves the // importer in hev1 mode where inline-SPS keyframes still self-initialize. let config = super::config_from_hvcc(hvcc_bytes)?; + // A new record starts a new coded sequence, whose HDR metadata the samples will carry. + self.catalog.hdr(&mut self.track, Default::default(), true)?; self.length_size = Some(super::Hvcc::parse(hvcc_bytes)?.length_size); self.apply_config(config)?; Ok(()) diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index d29437fe9c6..b2512002722 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -2367,14 +2367,24 @@ fn vp9_vpcc_carries_the_resolved_color() { }); config.coded_width = Some(64); config.coded_height = Some(64); - config.color = Some(color(9, 16, 9, false)); - let trak = super::synthesize_video_trak(1, 90_000, &config, None).unwrap(); - let trex = mp4_atom::Trex { - track_id: 1, - default_sample_description_index: 1, - ..Default::default() + let reimport = |resolved: hang::catalog::Color| { + let mut config = config.clone(); + config.color = Some(resolved); + let trak = super::synthesize_video_trak(1, 90_000, &config, None).unwrap(); + let trex = mp4_atom::Trex { + track_id: 1, + default_sample_description_index: 1, + ..Default::default() + }; + let catalog = run_fmp4(&super::encode_init(None, vec![trak], vec![trex]).unwrap()); + catalog.video.renditions.values().next().unwrap().color.clone() }; - let catalog = run_fmp4(&super::encode_init(None, vec![trak], vec![trex]).unwrap()); - let video = catalog.video.renditions.values().next().unwrap(); - assert_eq!(video.color, Some(color(9, 16, 9, false))); + + assert_eq!(reimport(color(9, 16, 9, false)), Some(color(9, 16, 9, false))); + + // A partial colour leaves the rest unspecified, never the placeholder's BT.709. + let mut primaries = hang::catalog::Color::default(); + primaries.primaries = Some(9); + let back = reimport(primaries).unwrap(); + assert_eq!((back.primaries, back.transfer, back.matrix), (Some(9), None, None)); } diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index 90f5a5c4e54..550f4b52d1f 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -785,16 +785,17 @@ fn color_colr(color: Option<&hang::catalog::Color>) -> Option { }) } -/// Overlay the catalog colour on a `vpcC`, so it never contradicts the `colr` box written beside -/// it; a VP9 codec string may still carry a container's placeholder BT.709. +/// Write the catalog colour into a `vpcC`, so it never contradicts the `colr` box written beside +/// it. A VP9 codec string may carry a container's placeholder BT.709, so once the catalog knows any +/// colour, the fields it leaves out are written unspecified (2) rather than kept. fn vpcc_color(mut vpcc: mp4_atom::VpcC, color: Option<&hang::catalog::Color>) -> mp4_atom::VpcC { let Some(color) = color else { return vpcc; }; - vpcc.color_primaries = color.primaries.unwrap_or(vpcc.color_primaries); - vpcc.transfer_characteristics = color.transfer.unwrap_or(vpcc.transfer_characteristics); - vpcc.matrix_coefficients = color.matrix.unwrap_or(vpcc.matrix_coefficients); - vpcc.video_full_range_flag = color.full_range.unwrap_or(vpcc.video_full_range_flag); + vpcc.color_primaries = color.primaries.unwrap_or(2); + vpcc.transfer_characteristics = color.transfer.unwrap_or(2); + vpcc.matrix_coefficients = color.matrix.unwrap_or(2); + vpcc.video_full_range_flag = color.full_range.unwrap_or(false); vpcc } From ed346c3e640fb6bd4b173cfad8a1fe7f6171612c Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 17 Oct 2026 00:51:31 -0700 Subject: [PATCH 11/14] fix(moq-mux): keep the code points of a QuickTime nclc colr box Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/fmp4/import_test.rs | 17 +++++++++ rs/moq-mux/src/container/fmp4/mod.rs | 40 ++++++++++++-------- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index b2512002722..4efccf73c41 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -2388,3 +2388,20 @@ fn vp9_vpcc_carries_the_resolved_color() { let back = reimport(primaries).unwrap(); assert_eq!((back.primaries, back.transfer, back.matrix), (Some(9), None, None)); } + +/// QuickTime's `nclc` carries the code points without a range, and keeps them; a code point past +/// 255 is refused. +#[test] +fn colr_nclc_keeps_its_code_points() { + let nclc = mp4_atom::Colr::Nclc { + colour_primaries: 9, + transfer_characteristics: 16, + matrix_coefficients: 9, + }; + let color = super::colr_color(&nclc).unwrap().unwrap(); + assert_eq!( + (color.primaries, color.transfer, color.matrix, color.full_range), + (Some(9), Some(16), Some(9), None) + ); + assert!(super::colr_color(&mp4_atom::Colr::new(300, 1, 1, false).unwrap()).is_err()); +} diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index 550f4b52d1f..b35fa458953 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -748,25 +748,35 @@ fn trun_duration(duration: Timestamp, timescale: moq_net::Timescale) -> Result Result> { - let mp4_atom::Colr::Nclx { - colour_primaries, - transfer_characteristics, - matrix_coefficients, - full_range_flag, - } = *colr - else { - return Ok(None); + let (primaries, transfer, matrix, full_range) = match *colr { + mp4_atom::Colr::Nclx { + colour_primaries, + transfer_characteristics, + matrix_coefficients, + full_range_flag, + } => ( + colour_primaries, + transfer_characteristics, + matrix_coefficients, + Some(full_range_flag), + ), + mp4_atom::Colr::Nclc { + colour_primaries, + transfer_characteristics, + matrix_coefficients, + } => (colour_primaries, transfer_characteristics, matrix_coefficients, None), + _ => return Ok(None), }; let code = |value: u16| u8::try_from(value).map_err(|_| Error::InvalidColr(value)); Ok(crate::codec::color::cicp( - code(colour_primaries)?, - code(transfer_characteristics)?, - code(matrix_coefficients)?, - Some(full_range_flag), + code(primaries)?, + code(transfer)?, + code(matrix)?, + full_range, )) } From ceab18f176dd1c1ba0b27c6979cf74a26af74bec Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 17 Oct 2026 01:24:41 -0700 Subject: [PATCH 12/14] fix(moq-mux): keep av1C HDR across the first sync sample, accept only registered colour The first sequence header after an av1C is the record's own, so it no longer resets the configOBUs' HDR metadata. Only registered H.273 code points count as signalled, and non-finite Matroska mastering values are refused. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/codec/av1/import.rs | 45 +++++++++++++++++++-- rs/moq-mux/src/codec/color.rs | 17 ++++---- rs/moq-mux/src/container/mkv/import.rs | 3 ++ rs/moq-mux/src/container/mkv/import_test.rs | 1 + 4 files changed, 56 insertions(+), 10 deletions(-) diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index 6f6518b32da..f256ecce0cb 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -109,9 +109,10 @@ impl Import { full_range: false, }); // configOBUs follow the four header bytes; samples need not repeat what they carry. A new - // record starts a new coded sequence, so earlier in-band HDR metadata no longer applies. + // record starts a new coded sequence, whose HDR metadata is what its configOBUs carry. config.color = color(&data[4..]); - self.catalog.hdr(&mut self.track, Default::default(), true)?; + let hdr = find_hdr(&Bytes::copy_from_slice(&data[4..])); + self.catalog.hdr(&mut self.track, hdr, true)?; self.apply_config(config)?; Ok(()) } @@ -248,7 +249,10 @@ impl Import { let seq = find_sequence_header(&frame.payload); // A new sequence header starts a coded sequence whose HDR metadata replaces the last // one's. It goes first, so a keyframe that also configures publishes once. - let sequence = seq.as_ref().is_some_and(|seq| self.last_seq.as_ref() != Some(seq)); + // The first header after an av1C is the record's own sequence, which sync samples repeat. + let sequence = seq + .as_ref() + .is_some_and(|seq| self.last_seq.as_ref().is_some_and(|last| last != seq)); self.catalog.hdr(&mut self.track, find_hdr(&frame.payload), sequence)?; if let Some(seq) = seq { self.configure_from_seq(&seq)?; @@ -410,5 +414,40 @@ mod tests { .expect("colour"); assert_eq!(color.content_light, crate::codec::color::test::hdr10().content_light); assert!(color.mastering.is_some()); + + // The first sync sample repeats a sequence header but not the metadata OBUs. + let sequence = av1_sequence_header(); + let pts = moq_net::Timestamp::from_micros(0).unwrap(); + let mut split = super::super::Split::new(); + let mut frames = split.decode(&sequence, pts).unwrap(); + frames.extend(split.flush(pts).unwrap()); + import.decode(frames).unwrap(); + let color = catalog.snapshot().video.renditions["video"] + .color + .clone() + .expect("colour"); + assert!( + color.mastering.is_some(), + "the av1C's HDR metadata survives the first sync sample" + ); + } + + /// A temporal unit of a temporal delimiter, the sequence header from `av1.mp4`'s av1C, and a frame. + fn av1_sequence_header() -> Vec { + let init = include_bytes!("../../container/fmp4/test_data/av1.mp4"); + let mut cursor = std::io::Cursor::new(&init[..]); + let moov = loop { + match ::decode(&mut cursor).unwrap() { + mp4_atom::Any::Moov(moov) => break moov, + _ => continue, + } + }; + let mp4_atom::Codec::Av01(av01) = &moov.trak[0].mdia.minf.stbl.stsd.codecs[0] else { + panic!("expected av01"); + }; + let mut tu = vec![0x12, 0x00]; + tu.extend_from_slice(&av01.av1c.config_obus); + tu.extend_from_slice(&[0x32, 0x01, 0x10]); + tu } } diff --git a/rs/moq-mux/src/codec/color.rs b/rs/moq-mux/src/codec/color.rs index 10da0a141b1..bd6b76c6949 100644 --- a/rs/moq-mux/src/codec/color.rs +++ b/rs/moq-mux/src/codec/color.rs @@ -12,14 +12,16 @@ use hang::catalog::{Chromaticity, Color, ContentLight, Mastering}; /// /// `None` when nothing is left, so a source that signals nothing adds no catalog field. pub(crate) fn cicp(primaries: u8, transfer: u8, matrix: u8, full_range: Option) -> Option { - // 2 is unspecified for all three and 3 is reserved; 0 is reserved for primaries and transfer - // but is the identity (RGB) matrix. Cameras ship zeroed descriptions, so a reserved value is - // treated as unsignalled rather than refusing the stream. - let known = |code: u8, zero: bool| (code != 2 && code != 3 && (zero || code != 0)).then_some(code); + // Only H.273's registered values, never 2 (unspecified). Cameras ship zeroed and other reserved + // descriptions, so a reserved value reads as unsignalled, letting a container fill it, rather + // than refusing the stream. + let registered = |code: u8, values: &[std::ops::RangeInclusive]| { + values.iter().any(|range| range.contains(&code)).then_some(code) + }; let mut color = Color::default(); - color.primaries = known(primaries, false); - color.transfer = known(transfer, false); - color.matrix = known(matrix, true); + color.primaries = registered(primaries, &[1..=1, 4..=12, 22..=22]); + color.transfer = registered(transfer, &[1..=1, 4..=18]); + color.matrix = registered(matrix, &[0..=1, 4..=14]); color.full_range = full_range; (color != Color::default()).then_some(color) } @@ -270,6 +272,7 @@ pub(crate) mod test { fn cicp_drops_unspecified_and_reserved() { assert_eq!(cicp(2, 2, 2, None), None); assert_eq!(cicp(0, 0, 3, None), None); + assert_eq!(cicp(255, 19, 15, None), None, "past the registered values"); let color = cicp(9, 16, 0, Some(false)).unwrap(); assert_eq!( (color.primaries, color.transfer, color.matrix, color.full_range), diff --git a/rs/moq-mux/src/container/mkv/import.rs b/rs/moq-mux/src/container/mkv/import.rs index fec670bd145..1c02bd527c4 100644 --- a/rs/moq-mux/src/container/mkv/import.rs +++ b/rs/moq-mux/src/container/mkv/import.rs @@ -761,6 +761,9 @@ pub(super) fn matroska_color(children: &[MatroskaSpec]) -> Result (9, x), _ => continue, }; + if !value.is_finite() { + return Err(Error::InvalidColour("mastering value").into()); + } v[i] = Some(*value); } let xy = |i: usize| Some(Chromaticity { x: v[i]?, y: v[i + 1]? }); diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index 8f0d22527c6..a3af0987b2a 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -1177,6 +1177,7 @@ fn unrepresentable_colour_is_refused() { MatroskaSpec::MaxCLL(70_000), MatroskaSpec::MaxFALL(65_536), MatroskaSpec::Primaries(300), + MatroskaSpec::MasteringMetadata(Master::Full(vec![MatroskaSpec::LuminanceMax(f64::NAN)])), ] { assert!(super::import::matroska_color(&[child]).is_err()); } From e7c3f329a95ae7495c50ec085b3e2b5e943c330d Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 17 Oct 2026 07:40:07 -0700 Subject: [PATCH 13/14] feat(hang)!: name the colour range `range`, matching moq-video The catalog's colour carries `range: "limited" | "full"` instead of `fullRange: bool`, mirroring moq-video's `Range` (#5808). Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-hang.md | 4 ++-- js/hang/src/catalog/video.test.ts | 4 +++- js/hang/src/catalog/video.ts | 4 ++-- rs/hang/src/catalog/video/color.rs | 18 ++++++++++++++---- rs/moq-mux/src/codec/color.rs | 16 ++++++++++------ rs/moq-mux/src/codec/h264/import.rs | 4 ++-- rs/moq-mux/src/codec/h264/mod.rs | 8 ++++---- rs/moq-mux/src/codec/h265/import.rs | 4 ++-- rs/moq-mux/src/codec/h265/mod.rs | 4 ++-- rs/moq-mux/src/container/fmp4/import_test.rs | 8 ++++++-- rs/moq-mux/src/container/fmp4/mod.rs | 6 +++--- rs/moq-mux/src/container/mkv/export.rs | 7 +++++-- rs/moq-mux/src/container/mkv/import_test.rs | 4 ++-- 13 files changed, 57 insertions(+), 34 deletions(-) diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index a8eb7780c3a..dfbb4dcebdc 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -252,7 +252,7 @@ type Color = { "primaries": number | undefined, "transfer": number | undefined, "matrix": number | undefined, - "fullRange": boolean | undefined, + "range": "limited" | "full" | undefined, "mastering": { "red": Chromaticity, "green": Chromaticity, @@ -270,7 +270,7 @@ type Color = { type Chromaticity = { "x": number, "y": number } ~~~ -`primaries`, `transfer`, and `matrix` are the ColourPrimaries, TransferCharacteristics, and MatrixCoefficients code points of {{h273}}, and `fullRange` is its VideoFullRangeFlag. +`primaries`, `transfer`, and `matrix` are the ColourPrimaries, TransferCharacteristics, and MatrixCoefficients code points of {{h273}}, and `range` is its VideoFullRangeFlag: `full` for 1 and `limited` for 0. `mastering` is the {{st2086}} mastering display: CIE 1931 xy chromaticities and luminance in candelas per square metre. `contentLight` is the {{cta861}} MaxCLL and MaxFALL in candelas per square metre, where 0 means unknown. A publisher MUST omit a field its source leaves unspecified rather than write a default, and SHOULD prefer the value signalled in the elementary stream over a container's when they disagree. diff --git a/js/hang/src/catalog/video.test.ts b/js/hang/src/catalog/video.test.ts index 447c4b68924..c32f1efc1dd 100644 --- a/js/hang/src/catalog/video.test.ts +++ b/js/hang/src/catalog/video.test.ts @@ -172,7 +172,7 @@ test("color parses the H.273 code points and HDR10 metadata a Rust publisher wri primaries: 9, transfer: 16, matrix: 9, - fullRange: false, + range: "limited", mastering: { red: { x: 0.708, y: 0.292 }, green: { x: 0.17, y: 0.797 }, @@ -195,4 +195,6 @@ test("color refuses out-of-range code points and light levels", () => { expect(parse({ primaries: 256 })).toBe(false); expect(parse({ contentLight: { maxCll: 65536, maxFall: 0 } })).toBe(false); expect(parse({ matrix: 0 })).toBe(true); + expect(parse({ range: "full" })).toBe(true); + expect(parse({ range: true })).toBe(false); }); diff --git a/js/hang/src/catalog/video.ts b/js/hang/src/catalog/video.ts index e2e01201ab3..fba7dc0f03d 100644 --- a/js/hang/src/catalog/video.ts +++ b/js/hang/src/catalog/video.ts @@ -27,8 +27,8 @@ export const ColorSchema = z.object({ transfer: z.optional(u8Schema), // H.273 MatrixCoefficients: 0 is RGB, 1 is BT.709, 9 is BT.2020 non-constant luminance. matrix: z.optional(u8Schema), - // True for full-range samples, false for limited (studio) range. - fullRange: z.optional(z.boolean()), + // The numeric range of the samples: studio swing, or every code used. + range: z.optional(z.enum(["limited", "full"])), // The display the content was mastered on, as SMPTE ST 2086, luminance in cd/m². mastering: z.optional( z.object({ diff --git a/rs/hang/src/catalog/video/color.rs b/rs/hang/src/catalog/video/color.rs index 4bcfe9461bf..957f77abc07 100644 --- a/rs/hang/src/catalog/video/color.rs +++ b/rs/hang/src/catalog/video/color.rs @@ -21,8 +21,8 @@ pub struct Color { /// The matrix between RGB and Y'CbCr, as H.273 `MatrixCoefficients` (0 is RGB, 1 is BT.709, 9 is BT.2020 NCL). pub matrix: Option, - /// True for full-range samples, false for limited (studio) range. - pub full_range: Option, + /// The numeric range of the samples. + pub range: Option, /// The display the content was mastered on, as SMPTE ST 2086. pub mastering: Option, @@ -31,6 +31,16 @@ pub struct Color { pub content_light: Option, } +/// The numeric range of the samples, as H.273's `VideoFullRangeFlag`. +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum Range { + /// Studio swing: 8-bit luma spans 16..235 and chroma 16..240. + Limited, + /// Full swing: every 8-bit code is used. + Full, +} + /// The colour volume of a mastering display, as SMPTE ST 2086. #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq)] #[serde(rename_all = "camelCase")] @@ -76,7 +86,7 @@ mod test { primaries: Some(9), transfer: Some(16), matrix: Some(9), - full_range: Some(false), + range: Some(Range::Limited), mastering: Some(Mastering { red: Chromaticity { x: 0.708, y: 0.292 }, green: Chromaticity { x: 0.17, y: 0.797 }, @@ -98,7 +108,7 @@ mod test { "primaries": 9, "transfer": 16, "matrix": 9, - "fullRange": false, + "range": "limited", "mastering": { "red": { "x": 0.708, "y": 0.292 }, "green": { "x": 0.17, "y": 0.797 }, diff --git a/rs/moq-mux/src/codec/color.rs b/rs/moq-mux/src/codec/color.rs index bd6b76c6949..cca902b4e4c 100644 --- a/rs/moq-mux/src/codec/color.rs +++ b/rs/moq-mux/src/codec/color.rs @@ -6,7 +6,7 @@ //! travels with the elementary stream, while a muxer adds the box and may get it wrong. So an //! importer resolves the bitstream's colour first and [`fill`]s only its gaps from the container. -use hang::catalog::{Chromaticity, Color, ContentLight, Mastering}; +use hang::catalog::{Chromaticity, Color, ContentLight, Mastering, Range}; /// The colour a set of H.273 code points signals, leaving each unspecified or reserved one absent. /// @@ -22,7 +22,7 @@ pub(crate) fn cicp(primaries: u8, transfer: u8, matrix: u8, full_range: Option, fallback: Option<&Color>, source: field("primaries", source, &mut color.primaries, &fallback.primaries); field("transfer", source, &mut color.transfer, &fallback.transfer); field("matrix", source, &mut color.matrix, &fallback.matrix); - field("full_range", source, &mut color.full_range, &fallback.full_range); + field("range", source, &mut color.range, &fallback.range); field("mastering", source, &mut color.mastering, &fallback.mastering); field( "content_light", @@ -275,8 +275,8 @@ pub(crate) mod test { assert_eq!(cicp(255, 19, 15, None), None, "past the registered values"); let color = cicp(9, 16, 0, Some(false)).unwrap(); assert_eq!( - (color.primaries, color.transfer, color.matrix, color.full_range), - (Some(9), Some(16), Some(0), Some(false)) + (color.primaries, color.transfer, color.matrix, color.range), + (Some(9), Some(16), Some(0), Some(hang::catalog::Range::Limited)) ); } @@ -288,7 +288,11 @@ pub(crate) mod test { assert_eq!(color.primaries, Some(1), "the stream wins a conflict"); assert_eq!(color.matrix, Some(1)); assert_eq!(color.transfer, Some(1), "the container fills an unspecified field"); - assert_eq!(color.full_range, Some(true), "the container fills an unsignalled range"); + assert_eq!( + color.range, + Some(hang::catalog::Range::Full), + "the container fills an unsignalled range" + ); } #[test] diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index 446db44bfed..d6bd6e28db5 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -336,8 +336,8 @@ mod tests { .clone() .expect("colour"); assert_eq!( - (color.primaries, color.transfer, color.matrix, color.full_range), - (Some(9), Some(16), Some(9), Some(false)) + (color.primaries, color.transfer, color.matrix, color.range), + (Some(9), Some(16), Some(9), Some(hang::catalog::Range::Limited)) ); let hdr = crate::codec::color::test::hdr10(); assert_eq!( diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index d27facc9956..ba8cd201d5b 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -798,13 +798,13 @@ mod tests { fn sps_color_reads_the_video_signal_type() { let sdr = sps_color(fixtures::SPS_BT709).unwrap(); assert_eq!( - (sdr.primaries, sdr.transfer, sdr.matrix, sdr.full_range), - (Some(1), Some(1), Some(1), Some(false)) + (sdr.primaries, sdr.transfer, sdr.matrix, sdr.range), + (Some(1), Some(1), Some(1), Some(hang::catalog::Range::Limited)) ); let hdr = sps_color(fixtures::SPS_HDR10).unwrap(); assert_eq!( - (hdr.primaries, hdr.transfer, hdr.matrix, hdr.full_range), - (Some(9), Some(16), Some(9), Some(false)) + (hdr.primaries, hdr.transfer, hdr.matrix, hdr.range), + (Some(9), Some(16), Some(9), Some(hang::catalog::Range::Limited)) ); assert_eq!(sps_color(fixtures::SPS_IPB), None, "a VUI without a video signal type"); } diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index b6036e5bde6..7f0d5ff0c55 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -406,8 +406,8 @@ mod tests { .clone() .expect("colour"); assert_eq!( - (color.primaries, color.transfer, color.matrix, color.full_range), - (Some(9), Some(16), Some(9), Some(false)) + (color.primaries, color.transfer, color.matrix, color.range), + (Some(9), Some(16), Some(9), Some(hang::catalog::Range::Limited)) ); let hdr = crate::codec::color::test::hdr10(); assert_eq!( diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index b6ee087e629..3d1d35da6bc 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -716,8 +716,8 @@ mod tests { }; let color = config_from_hvcc(&hvcc(fixtures::SPS_HDR10)).unwrap().color.unwrap(); assert_eq!( - (color.primaries, color.transfer, color.matrix, color.full_range), - (Some(9), Some(16), Some(9), Some(false)) + (color.primaries, color.transfer, color.matrix, color.range), + (Some(9), Some(16), Some(9), Some(hang::catalog::Range::Limited)) ); assert_eq!(hvcc_color(&hvcc(fixtures::SPS)), None); assert_eq!(hvcc_color(&hvcc(fixtures::SPS_ZERO_COLOUR)), None); diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index 4efccf73c41..bf7e8510907 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -271,7 +271,10 @@ fn test_av1_catalog() { assert!(video.codec.to_string().starts_with("av01."), "codec: {}", video.codec); // The av1C's sequence header leaves the code points unspecified, but AV1 always states the range. let color = video.color.as_ref().expect("the range"); - assert_eq!((color.primaries, color.full_range), (None, Some(false))); + assert_eq!( + (color.primaries, color.range), + (None, Some(hang::catalog::Range::Limited)) + ); assert!(matches!(video.container, Container::Cmaf { .. })); let Container::Cmaf { init, .. } = &video.container else { @@ -2234,6 +2237,7 @@ async fn ttml_body_begins_at_the_serialized_frame_timestamp() { payload.contains(r#""#), "{payload}" ); +} /// An avc1 init whose avcC carries `sps` and whose sample entry carries the `colr` box for `color`. fn avc1_init(sps: &'static [u8], color: Option) -> bytes::Bytes { @@ -2400,7 +2404,7 @@ fn colr_nclc_keeps_its_code_points() { }; let color = super::colr_color(&nclc).unwrap().unwrap(); assert_eq!( - (color.primaries, color.transfer, color.matrix, color.full_range), + (color.primaries, color.transfer, color.matrix, color.range), (Some(9), Some(16), Some(9), None) ); assert!(super::colr_color(&mp4_atom::Colr::new(300, 1, 1, false).unwrap()).is_err()); diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index b35fa458953..86d58a75758 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -784,14 +784,14 @@ pub(crate) fn colr_color(colr: &mp4_atom::Colr) -> Result) -> Option { let color = color?; - if color.primaries.is_none() && color.transfer.is_none() && color.matrix.is_none() && color.full_range.is_none() { + if color.primaries.is_none() && color.transfer.is_none() && color.matrix.is_none() && color.range.is_none() { return None; } Some(mp4_atom::Colr::Nclx { colour_primaries: color.primaries.unwrap_or(2).into(), transfer_characteristics: color.transfer.unwrap_or(2).into(), matrix_coefficients: color.matrix.unwrap_or(2).into(), - full_range_flag: color.full_range.unwrap_or(false), + full_range_flag: color.range == Some(hang::catalog::Range::Full), }) } @@ -805,7 +805,7 @@ fn vpcc_color(mut vpcc: mp4_atom::VpcC, color: Option<&hang::catalog::Color>) -> vpcc.color_primaries = color.primaries.unwrap_or(2); vpcc.transfer_characteristics = color.transfer.unwrap_or(2); vpcc.matrix_coefficients = color.matrix.unwrap_or(2); - vpcc.video_full_range_flag = color.full_range.unwrap_or(false); + vpcc.video_full_range_flag = color.range == Some(hang::catalog::Range::Full); vpcc } diff --git a/rs/moq-mux/src/container/mkv/export.rs b/rs/moq-mux/src/container/mkv/export.rs index 96636e272f1..eb9ba7425c0 100644 --- a/rs/moq-mux/src/container/mkv/export.rs +++ b/rs/moq-mux/src/container/mkv/export.rs @@ -734,8 +734,11 @@ pub(super) fn matroska_colour(color: &hang::catalog::Color) -> Option 1, + hang::catalog::Range::Full => 2, + })); } if let Some(v) = color.transfer { children.push(MatroskaSpec::TransferCharacteristics(v.into())); diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index a3af0987b2a..1dc76a32685 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -1147,8 +1147,8 @@ fn sps_color_outranks_colour_which_fills_its_gaps() { let catalog = run(&data); let color = catalog.video.renditions.values().next().unwrap().color.clone().unwrap(); assert_eq!( - (color.primaries, color.transfer, color.matrix, color.full_range), - (Some(1), Some(1), Some(1), Some(false)) + (color.primaries, color.transfer, color.matrix, color.range), + (Some(1), Some(1), Some(1), Some(hang::catalog::Range::Limited)) ); assert_eq!( (color.mastering, color.content_light), From 1b4ffcab65085349a69c9ac49b935a0e31621d65 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sat, 17 Oct 2026 08:44:42 -0700 Subject: [PATCH 14/14] fix(moq-mux): accept H.273 matrix 15-17, reset HDR on a new seed sequence, range-check Matroska mastering Matrix coefficients 15-17 (IPT-C2, YCgCo-Re, YCgCo-Ro) are registered, so they count as signalled. Seeding an Annex-B or raw-OBU importer with a new SPS or sequence header drops the last sequence's HDR metadata before the config publishes. Matroska mastering chromaticities outside 0..=1 and negative luminances are refused. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/codec/av1/import.rs | 26 +++++++++- rs/moq-mux/src/codec/color.rs | 8 +++- rs/moq-mux/src/codec/h264/import.rs | 46 +++++++++++++++++- rs/moq-mux/src/codec/h265/import.rs | 53 ++++++++++++++++++++- rs/moq-mux/src/container/mkv/import.rs | 7 ++- rs/moq-mux/src/container/mkv/import_test.rs | 2 + 6 files changed, 134 insertions(+), 8 deletions(-) diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index f256ecce0cb..324e33e9a32 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -79,8 +79,12 @@ impl Import { return Ok(()); } - // Raw OBUs: resolve the config from any sequence header. + // Raw OBUs: resolve the config from any sequence header. A new one starts a coded sequence, + // so the last one's HDR metadata is dropped before the config publishes. if let Some(seq) = find_sequence_header(data) { + if self.last_seq.as_ref() != Some(&seq) { + self.catalog.hdr(&mut self.track, Default::default(), true)?; + } self.configure_from_seq(&seq)?; } Ok(()) @@ -432,6 +436,26 @@ mod tests { ); } + /// Seeding with raw OBUs whose sequence header is new starts a new coded sequence, dropping the + /// HDR metadata the last one carried. + #[tokio::test(start_paused = true)] + async fn a_new_raw_sequence_drops_stale_hdr_metadata() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let track = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let mut import = Import::new(track, catalog.reserve(), Default::default()).unwrap(); + let mut av1c = vec![0x81, 0x08, 0x0c, 0x00]; + av1c.extend(hdr10_obus()); + import.initialize(&av1c).unwrap(); + let color = || catalog.snapshot().video.renditions["video"].color.clone().unwrap(); + assert!(color().mastering.is_some()); + + import.initialize(&av1_sequence_header()).unwrap(); + assert_eq!((color().mastering, color().content_light), (None, None)); + } + /// A temporal unit of a temporal delimiter, the sequence header from `av1.mp4`'s av1C, and a frame. fn av1_sequence_header() -> Vec { let init = include_bytes!("../../container/fmp4/test_data/av1.mp4"); diff --git a/rs/moq-mux/src/codec/color.rs b/rs/moq-mux/src/codec/color.rs index cca902b4e4c..92da681ed3f 100644 --- a/rs/moq-mux/src/codec/color.rs +++ b/rs/moq-mux/src/codec/color.rs @@ -21,7 +21,7 @@ pub(crate) fn cicp(primaries: u8, transfer: u8, matrix: u8, full_range: Option Result<()> { + if self.last_sps.as_ref() != Some(sps) { + self.catalog.hdr(&mut self.track, Default::default(), true)?; + } + self.configure_from_sps(sps)?; + Ok(()) + } + /// The MoQ track name this importer publishes on. pub fn name(&self) -> &str { self.track.name() @@ -411,6 +422,37 @@ mod tests { assert_eq!((color().mastering, color().content_light), (None, None)); } + /// Seeding an Annex-B importer with a new SPS starts a new coded sequence, dropping the last + /// one's HDR metadata before the new config publishes. + #[tokio::test(start_paused = true)] + async fn a_new_seed_sps_drops_stale_hdr_metadata() { + use crate::codec::h264::fixtures::{PPS, SEI_CONTENT_LIGHT, SEI_MASTERING, SPS_BT709, SPS_HDR10}; + + let annexb = |nals: &[&[u8]]| { + let mut buf = BytesMut::new(); + for nal in nals { + buf.extend_from_slice(&[0, 0, 0, 1]); + buf.extend_from_slice(nal); + } + buf + }; + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.reserve(), Default::default()).unwrap(); + let mut split = Split::new(); + let pts = moq_net::Timestamp::from_micros(0).unwrap(); + let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; + let au = annexb(&[SPS_HDR10, PPS, SEI_MASTERING, SEI_CONTENT_LIGHT, idr]); + let mut frames = split.decode(&au, pts).unwrap(); + frames.extend(split.flush(pts).unwrap()); + import.decode(frames).unwrap(); + let color = || catalog.snapshot().video.renditions["video"].color.clone().unwrap(); + assert!(color().mastering.is_some()); + + import.initialize(&annexb(&[SPS_BT709, PPS])).unwrap(); + assert_eq!(color().primaries, Some(1)); + assert_eq!((color().mastering, color().content_light), (None, None)); + } + /// The stream outranks the caller's hint, which only fills what the stream leaves unsaid. #[tokio::test(start_paused = true)] async fn stream_color_outranks_the_hint() { diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index 7f0d5ff0c55..fea14f83426 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -103,17 +103,28 @@ impl Import { let mut nals = NalIterator::new(&mut scan); while let Some(nal) = nals.next().transpose()? { if is_sps(&nal) { - self.configure_from_sps(&nal)?; + self.seed_sps(&nal)?; } } if let Some(nal) = nals.flush()? && is_sps(&nal) { - self.configure_from_sps(&nal)?; + self.seed_sps(&nal)?; } Ok(()) } + /// Configure from an SPS in a seed buffer. A new one starts a coded sequence, so the last one's + /// HDR metadata is dropped before the config publishes; the seed's own SEI arrives with its + /// keyframe through the splitter. + fn seed_sps(&mut self, sps: &Bytes) -> Result<()> { + if self.last_sps.as_ref() != Some(sps) { + self.catalog.hdr(&mut self.track, Default::default(), true)?; + } + self.configure_from_sps(sps)?; + Ok(()) + } + /// The MoQ track name this importer publishes on. pub fn name(&self) -> &str { self.track.name() @@ -376,6 +387,44 @@ mod tests { use super::*; use crate::codec::h265::{Split, fixtures}; + /// Seeding an Annex-B importer with a new SPS starts a new coded sequence, dropping the last + /// one's HDR metadata before the new config publishes. + #[tokio::test(start_paused = true)] + async fn a_new_seed_sps_drops_stale_hdr_metadata() { + let annexb = |nals: &[&[u8]]| { + let mut buf = BytesMut::new(); + for nal in nals { + buf.extend_from_slice(&[0, 0, 0, 1]); + buf.extend_from_slice(nal); + } + buf + }; + let (track, catalog) = setup("video"); + let mut import = Import::new(track, catalog.reserve(), Default::default()).unwrap(); + let mut split = Split::new(); + let pts = moq_net::Timestamp::from_micros(0).unwrap(); + let idr: &[u8] = &[0x26, 0x01, 0x80, 0xaa]; // IdrWRadl (19) + let au = annexb(&[ + fixtures::VPS, + fixtures::SPS_HDR10, + fixtures::PPS, + fixtures::SEI_CONTENT_LIGHT, + fixtures::SEI_MASTERING, + idr, + ]); + let mut frames = split.decode(&au, pts).unwrap(); + frames.extend(split.flush(pts).unwrap()); + import.decode(frames).unwrap(); + let color = || catalog.snapshot().video.renditions["video"].color.clone(); + assert!(color().unwrap().mastering.is_some()); + + // An SPS that signals no colour: nothing is left once the stale metadata goes. + import + .initialize(&annexb(&[fixtures::VPS, fixtures::SPS, fixtures::PPS])) + .unwrap(); + assert_eq!(color(), None); + } + /// An HDR10 keyframe publishes its VUI colour and the static HDR metadata its SEI carries. #[tokio::test(start_paused = true)] async fn hev1_keyframe_publishes_hdr10_color() { diff --git a/rs/moq-mux/src/container/mkv/import.rs b/rs/moq-mux/src/container/mkv/import.rs index 1c02bd527c4..540fb425f05 100644 --- a/rs/moq-mux/src/container/mkv/import.rs +++ b/rs/moq-mux/src/container/mkv/import.rs @@ -761,7 +761,12 @@ pub(super) fn matroska_color(children: &[MatroskaSpec]) -> Result (9, x), _ => continue, }; - if !value.is_finite() { + // Chromaticities lie in 0..=1 and luminances are nonnegative (Matroska's ranges). + let valid = match i { + 0..=7 => (0.0..=1.0).contains(value), + _ => value.is_finite() && *value >= 0.0, + }; + if !valid { return Err(Error::InvalidColour("mastering value").into()); } v[i] = Some(*value); diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index 1dc76a32685..f09110e5a58 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -1178,6 +1178,8 @@ fn unrepresentable_colour_is_refused() { MatroskaSpec::MaxFALL(65_536), MatroskaSpec::Primaries(300), MatroskaSpec::MasteringMetadata(Master::Full(vec![MatroskaSpec::LuminanceMax(f64::NAN)])), + MatroskaSpec::MasteringMetadata(Master::Full(vec![MatroskaSpec::LuminanceMin(-1.0)])), + MatroskaSpec::MasteringMetadata(Master::Full(vec![MatroskaSpec::PrimaryRChromaticityX(1.5)])), ] { assert!(super::import::matroska_color(&[child]).is_err()); }