diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index ee9b49a6a94..feb9d7eb556 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -103,7 +103,6 @@ remain with their current quests. - [Plan: semantic fidelity](/quest/m1/ts-reliability/fidelity.md) - explicit profiles and PMT lifecycle contracts produce implementation quests - [Section placement](/quest/m1/ts-reliability/section-placement.md) - sections share the PCR placement clock instead of the video PTS - [Sequence holes get a time budget](/quest/m1/ts-reliability/hole-budget.md) - a hole gives up on max(wall, pts), so a break marker behind it on a paused track is delivered -- [AUD first](/quest/m1/ts-reliability/aud-first.md) - a keyframe's access unit delimiter stays ahead of the re-injected parameter sets - [TS source front end](/quest/m1/ts-reliability/source-front-end.md) - cli.md says to pipe a TS source's bytes, and the first PMT logs what each stream became - [Loss and recovery tests](/quest/m1/ts-reliability/recovery.md) - actual MoQ omissions exercise reacquisition and unaffected tracks diff --git a/quest/m1/ts-reliability/aud-first.md b/quest/m1/ts-reliability/aud-first.md deleted file mode 100644 index 3ed6e0768b1..00000000000 --- a/quest/m1/ts-reliability/aud-first.md +++ /dev/null @@ -1,28 +0,0 @@ -# [XS] The access unit delimiter stays first - -## Goal - -When a keyframe's payload starts with an access unit delimiter, every Annex B -writer emits the AUD, then the re-injected parameter sets, then the rest, as -H.264 §7.4.1.2.3 and H.265 §7.4.2.4.4 require. Today `export ts` writes SPS, -PPS, AUD, IDR. - -## Plan - -Decided 2026-10-19 by the maintainer (#5951): - -- Fix both writers: `video_es_payload` in - `rs/moq-mux/src/container/ts/export.rs` and `annexb::from_length_prefixed` - in `rs/moq-mux/src/codec/annexb.rs`. The second also feeds the H.264 and - H.265 elementary exporters, `moq-rtc`, and the `moq-video` decoder, which - all get the right order. -- An AUD is H.264 NAL type 9 or H.265 type 35. A payload without one is - unchanged. -- Regression test: an AUD + IDR keyframe exports as AUD, SPS, PPS, IDR. It - fails today. - -Public API: none. - -## Closes - -- [#5951](https://github.com/moq-dev/moq/issues/5951) - export ts writes keyframe parameter sets ahead of the AUD diff --git a/rs/moq-mux/src/codec/annexb.rs b/rs/moq-mux/src/codec/annexb.rs index e2d981ec013..4f51fe6f592 100644 --- a/rs/moq-mux/src/codec/annexb.rs +++ b/rs/moq-mux/src/codec/annexb.rs @@ -50,11 +50,25 @@ pub(crate) fn length_prefixed_nals(data: &[u8], length_size: usize) -> Result { data: &'a [u8], length_size: usize, } +impl<'a> LengthPrefixedNals<'a> { + /// Take a leading access unit delimiter, which must stay ahead of injected parameter sets. + pub(crate) fn take_delimiter(&mut self, codec: InBandCodec) -> Result> { + if let Some(nal) = self.clone().next().transpose()? + && !nal.is_empty() + && codec.nal_type(nal) == codec.delimiter() + { + return self.next().transpose(); + } + Ok(None) + } +} + impl<'a> Iterator for LengthPrefixedNals<'a> { type Item = Result<&'a [u8]>; @@ -79,19 +93,29 @@ impl<'a> Iterator for LengthPrefixedNals<'a> { } /// Convert a length-prefixed NALU payload (avc1 / hvc1 wire shape) to Annex-B, -/// optionally prepending `prefix` bytes (typically VPS/SPS/PPS NAL units already -/// in Annex-B form, for keyframe parameter-set injection). +/// injecting `prefix` bytes (typically VPS/SPS/PPS NAL units already in Annex-B +/// form) after a leading access unit delimiter, or before the other NAL units. pub fn from_length_prefixed(payload: &[u8], length_size: usize, prefix: Option<&[u8]>) -> Result { let mut out = BytesMut::with_capacity(payload.len() + prefix.map(|p| p.len()).unwrap_or(0) + 16); - if let Some(p) = prefix { - out.extend_from_slice(p); + let mut nals = length_prefixed_nals(payload, length_size)?; + if let Some(prefix) = prefix { + let codec = after_start_code(prefix) + .ok() + .flatten() + .and_then(|start| prefix.get(start)) + .and_then(|&header| InBandCodec::of_parameter_set(header)); + if let Some(codec) = codec + && let Some(aud) = nals.take_delimiter(codec)? + { + out.extend_from_slice(&START_CODE); + out.extend_from_slice(aud); + } + out.extend_from_slice(prefix); } - - for nal in length_prefixed_nals(payload, length_size)? { + for nal in nals { out.extend_from_slice(&START_CODE); out.extend_from_slice(nal?); } - Ok(out.freeze()) } @@ -139,6 +163,18 @@ impl InBandCodec { } } + /// The codec whose parameter-set NAL begins with `header`. + /// + /// NAL headers overlap between codecs (an H.265 slice can look like an H.264 AUD), so + /// only a parameter set identifies one. + fn of_parameter_set(header: u8) -> Option { + match (header & 0x1f, (header >> 1) & 0x3f) { + (7 | 8, _) => Some(Self::H264), + (_, 32..=34) => Some(Self::H265), + _ => None, + } + } + fn delimiter(self) -> u8 { match self { Self::H264 => 9, @@ -412,21 +448,60 @@ mod tests { } #[test] - fn from_length_prefixed_multiple_nals_with_prefix() { - // Two NALs in one frame: AUD then IDR slice. Prefix gets prepended once. - let prefix = build_prefix([&Bytes::from_static(&[0x67, 0x42])]); - let payload = &[ - 0, 0, 0, 2, 0x09, 0x10, // AUD - 0, 0, 0, 2, 0x65, 0x88, // IDR slice - ]; - let out = from_length_prefixed(payload, 4, Some(&prefix)).unwrap(); + fn from_length_prefixed_keeps_aud_first() { + for (params, aud, slice) in [ + ( + vec![vec![0x67, 0x42], vec![0x68, 0xce]], + vec![0x09, 0x10], + vec![0x65, 0x88], + ), + ( + vec![vec![0x40, 1], vec![0x42, 1], vec![0x44, 1]], + vec![0x46, 1, 0x10], + vec![0x26, 1, 0x88], + ), + ] { + let params: Vec = params.into_iter().map(Bytes::from).collect(); + let prefix = build_prefix(¶ms); + for length_size in 1..=4 { + for leading in [None, Some(aud.as_slice())] { + let mut payload = Vec::new(); + for nal in leading.into_iter().chain([slice.as_slice()]) { + payload.extend_from_slice(&(nal.len() as u32).to_be_bytes()[4 - length_size..]); + payload.extend_from_slice(nal); + } + let mut expected = Vec::new(); + if let Some(aud) = leading { + expected.extend_from_slice(&START_CODE); + expected.extend_from_slice(aud); + } + expected.extend_from_slice(&prefix); + expected.extend_from_slice(&START_CODE); + expected.extend_from_slice(&slice); + let out = from_length_prefixed(&payload, length_size, Some(&prefix)).unwrap(); + assert_eq!(out.as_ref(), expected.as_slice()); + let mut without_prefix = Vec::new(); + for nal in leading.into_iter().chain([slice.as_slice()]) { + without_prefix.extend_from_slice(&START_CODE); + without_prefix.extend_from_slice(nal); + } + assert_eq!( + from_length_prefixed(&payload, length_size, None).unwrap(), + without_prefix + ); + } + } + } + } - // Single prefix followed by both NALs in Annex-B order. - let mut expected = Vec::new(); - expected.extend_from_slice(&prefix); - expected.extend_from_slice(&[0, 0, 0, 1, 0x09, 0x10]); // AUD - expected.extend_from_slice(&[0, 0, 0, 1, 0x65, 0x88]); // IDR - assert_eq!(out.as_ref(), expected.as_slice()); + #[test] + fn from_length_prefixed_distinguishes_codec_headers() { + // H.265 TSA_N with nuh_layer_id 32 has the same first byte as an H.264 AUD. + let prefix = build_prefix([&Bytes::from_static(&[0x40, 1])]); + let payload = &[0, 0, 0, 3, 0x09, 1, 0x80]; + let mut expected = prefix.to_vec(); + expected.extend_from_slice(&[0, 0, 0, 1, 0x09, 1, 0x80]); + assert_eq!(from_length_prefixed(payload, 4, Some(&prefix)).unwrap(), expected); } #[test] diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index bed85ac1c35..78b107c3dd1 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -43,6 +43,7 @@ use moq_net::Timestamp; use super::catalog::Catalog as _; use crate::catalog::Stream; use crate::catalog::hang::Catalog; +use crate::codec::annexb::InBandCodec; use crate::codec::video::{Hrd, Reorder}; use crate::codec::{aac, annexb, opus}; use crate::container::decode::{DecodeClock, MAX_REORDER}; @@ -3209,25 +3210,33 @@ fn ac3_sync_prefix(bytes: &[u8]) -> bool { } /// Build the Annex-B elementary-stream payload for one video frame: rewrite the -/// length-prefixed NALs to start-code-delimited NALs, prepending the parameter -/// sets (SPS/PPS, plus VPS for H.265) from the avcC/hvcC on keyframes so a -/// receiver can tune in mid-stream. +/// length-prefixed NALs to start-code-delimited NALs, injecting the parameter +/// sets (SPS/PPS, plus VPS for H.265) from the avcC/hvcC after a leading AUD on +/// keyframes so a receiver can tune in mid-stream. fn video_es_payload(stream_type: StreamType, description: Option<&Bytes>, frame: &Frame) -> anyhow::Result> { let description = description.context("video codec config (avcC/hvcC) not resolved")?; - let (length_size, params) = match stream_type { - StreamType::H264 => crate::codec::h264::avcc_params(description)?, - StreamType::H265 => crate::codec::h265::hvcc_params(description)?, + let (codec, (length_size, params)) = match stream_type { + StreamType::H264 => (InBandCodec::H264, crate::codec::h264::avcc_params(description)?), + StreamType::H265 => (InBandCodec::H265, crate::codec::h265::hvcc_params(description)?), other => anyhow::bail!("unsupported TS video stream type {other:?}"), }; let mut out = Vec::with_capacity(frame.payload.len() + 64); + let mut nals = annexb::length_prefixed_nals(&frame.payload, length_size)?; if frame.keyframe { + if let Some(aud) = nals.take_delimiter(codec)? { + out.extend_from_slice(&annexb::START_CODE); + out.extend_from_slice(aud); + } for nal in ¶ms { out.extend_from_slice(&annexb::START_CODE); out.extend_from_slice(nal); } } - annexb::length_prefixed_to_annexb(&frame.payload, length_size, &mut out)?; + for nal in nals { + out.extend_from_slice(&annexb::START_CODE); + out.extend_from_slice(nal?); + } Ok(out) } @@ -3458,6 +3467,68 @@ mod tests { Timestamp::from_millis(value).unwrap() } + #[test] + fn video_es_payload_keeps_aud_first() { + for (stream_type, params, aud, slice) in [ + ( + super::StreamType::H264, + vec![vec![0x67, 0x42, 0, 0x1f], vec![0x68, 0xce]], + vec![0x09, 0x10], + vec![0x65, 0x88], + ), + ( + super::StreamType::H265, + vec![vec![0x40, 1], vec![0x42, 1], vec![0x44, 1]], + vec![0x46, 1, 0x10], + vec![0x26, 1, 0x88], + ), + ] { + let params: Vec = params.into_iter().map(bytes::Bytes::from).collect(); + let description = if stream_type == super::StreamType::H264 { + crate::codec::h264::build_avcc(¶ms[..1], ¶ms[1..]).unwrap() + } else { + let mut hvcc = vec![0; 23]; + hvcc[21] = 3; + hvcc[22] = 3; + for (kind, nal) in (32..=34).zip(¶ms) { + hvcc.extend_from_slice(&[kind, 0, 1]); + hvcc.extend_from_slice(&(nal.len() as u16).to_be_bytes()); + hvcc.extend_from_slice(nal); + } + hvcc.into() + }; + for keyframe in [false, true] { + for leading in [None, Some(aud.as_slice())] { + let mut payload = Vec::new(); + for nal in leading.into_iter().chain([slice.as_slice()]) { + payload.extend_from_slice(&(nal.len() as u32).to_be_bytes()); + payload.extend_from_slice(nal); + } + let frame = super::Frame { + timestamp: ms(0), + payload: payload.into(), + keyframe, + duration: None, + }; + let mut expected = Vec::new(); + if let Some(aud) = leading { + expected.extend_from_slice(&super::annexb::START_CODE); + expected.extend_from_slice(aud); + } + if keyframe { + expected.extend_from_slice(&super::annexb::build_prefix(¶ms)); + } + expected.extend_from_slice(&super::annexb::START_CODE); + expected.extend_from_slice(&slice); + assert_eq!( + super::video_es_payload(stream_type, Some(&description), &frame).unwrap(), + expected + ); + } + } + } + } + /// A six-block E-AC-3 sync frame of 512 bytes: `stream` is byte 2's top bits (strmtyp, /// substreamid) and `rate` byte 4 (fscod, numblkscod, acmod, lfeon). fn eac3(stream: u8, rate: u8) -> Vec {