From 9a425f8917da9a7dca11c56622e0c7ccead5543e Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:37:48 -0700 Subject: [PATCH 1/3] chore: claim AUD-first quest From c92469794859dd624c45d472d9e2125ef1db48f7 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:42:33 -0700 Subject: [PATCH 2/3] fix(mux): keep access unit delimiters before injected parameter sets Co-Authored-By: GPT-6.1-Sol --- quest/m1/ts-reliability/README.md | 1 - quest/m1/ts-reliability/aud-first.md | 28 ------- rs/moq-mux/src/codec/annexb.rs | 109 +++++++++++++++++++++----- rs/moq-mux/src/container/ts/export.rs | 78 +++++++++++++++++- 4 files changed, 162 insertions(+), 54 deletions(-) delete mode 100644 quest/m1/ts-reliability/aud-first.md diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index 3d6e656cf6b..963632bb63f 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -104,7 +104,6 @@ remain with their current quests. - [Whole-PES import](/quest/m1/ts-reliability/pes-import.md) - private PES travels whole at its source placement, a missing PTS stays missing, and the `pes` framing is deleted - [Section placement](/quest/m1/ts-reliability/section-placement.md) - sections share the PCR placement clock instead of the video PTS - [Sequence holes get a time budget](/quest/m1/ts-reliability/hole-budget.md) - a hole gives up on max(wall, pts), so a break marker behind it on a paused track is delivered -- [AUD first](/quest/m1/ts-reliability/aud-first.md) - a keyframe's access unit delimiter stays ahead of the re-injected parameter sets - [TS source front end](/quest/m1/ts-reliability/source-front-end.md) - cli.md says to pipe a TS source's bytes, and the first PMT logs what each stream became - [Loss and recovery tests](/quest/m1/ts-reliability/recovery.md) - actual MoQ omissions exercise reacquisition and unaffected tracks diff --git a/quest/m1/ts-reliability/aud-first.md b/quest/m1/ts-reliability/aud-first.md deleted file mode 100644 index 3ed6e0768b1..00000000000 --- a/quest/m1/ts-reliability/aud-first.md +++ /dev/null @@ -1,28 +0,0 @@ -# [XS] The access unit delimiter stays first - -## Goal - -When a keyframe's payload starts with an access unit delimiter, every Annex B -writer emits the AUD, then the re-injected parameter sets, then the rest, as -H.264 §7.4.1.2.3 and H.265 §7.4.2.4.4 require. Today `export ts` writes SPS, -PPS, AUD, IDR. - -## Plan - -Decided 2026-10-19 by the maintainer (#5951): - -- Fix both writers: `video_es_payload` in - `rs/moq-mux/src/container/ts/export.rs` and `annexb::from_length_prefixed` - in `rs/moq-mux/src/codec/annexb.rs`. The second also feeds the H.264 and - H.265 elementary exporters, `moq-rtc`, and the `moq-video` decoder, which - all get the right order. -- An AUD is H.264 NAL type 9 or H.265 type 35. A payload without one is - unchanged. -- Regression test: an AUD + IDR keyframe exports as AUD, SPS, PPS, IDR. It - fails today. - -Public API: none. - -## Closes - -- [#5951](https://github.com/moq-dev/moq/issues/5951) - export ts writes keyframe parameter sets ahead of the AUD diff --git a/rs/moq-mux/src/codec/annexb.rs b/rs/moq-mux/src/codec/annexb.rs index e2d981ec013..bee47206b53 100644 --- a/rs/moq-mux/src/codec/annexb.rs +++ b/rs/moq-mux/src/codec/annexb.rs @@ -50,11 +50,31 @@ pub(crate) fn length_prefixed_nals(data: &[u8], length_size: usize) -> Result { data: &'a [u8], length_size: usize, } +impl<'a> LengthPrefixedNals<'a> { + /// Take a leading AUD, identifying the codec from an injected parameter-set header. + pub(crate) fn take_delimiter(&mut self, parameter_header: Option) -> Result> { + // NAL headers overlap between codecs: an H.265 slice can look like an H.264 AUD. + let codec = parameter_header.and_then(|header| match (header & 0x1f, (header >> 1) & 0x3f) { + (7 | 8, _) => Some(InBandCodec::H264), + (_, 32..=34) => Some(InBandCodec::H265), + _ => None, + }); + if let Some(nal) = self.clone().next().transpose()? + && !nal.is_empty() + && codec.is_some_and(|codec| codec.nal_type(nal) == codec.delimiter()) + { + return self.next().transpose(); + } + Ok(None) + } +} + impl<'a> Iterator for LengthPrefixedNals<'a> { type Item = Result<&'a [u8]>; @@ -79,19 +99,27 @@ impl<'a> Iterator for LengthPrefixedNals<'a> { } /// Convert a length-prefixed NALU payload (avc1 / hvc1 wire shape) to Annex-B, -/// optionally prepending `prefix` bytes (typically VPS/SPS/PPS NAL units already -/// in Annex-B form, for keyframe parameter-set injection). +/// injecting `prefix` bytes (typically VPS/SPS/PPS NAL units already in Annex-B +/// form) after a leading access unit delimiter, or before the other NAL units. pub fn from_length_prefixed(payload: &[u8], length_size: usize, prefix: Option<&[u8]>) -> Result { let mut out = BytesMut::with_capacity(payload.len() + prefix.map(|p| p.len()).unwrap_or(0) + 16); - if let Some(p) = prefix { - out.extend_from_slice(p); + let mut nals = length_prefixed_nals(payload, length_size)?; + if let Some(prefix) = prefix { + let header = after_start_code(prefix) + .ok() + .flatten() + .and_then(|start| prefix.get(start)) + .copied(); + if let Some(aud) = nals.take_delimiter(header)? { + out.extend_from_slice(&START_CODE); + out.extend_from_slice(aud); + } + out.extend_from_slice(prefix); } - - for nal in length_prefixed_nals(payload, length_size)? { + for nal in nals { out.extend_from_slice(&START_CODE); out.extend_from_slice(nal?); } - Ok(out.freeze()) } @@ -412,21 +440,60 @@ mod tests { } #[test] - fn from_length_prefixed_multiple_nals_with_prefix() { - // Two NALs in one frame: AUD then IDR slice. Prefix gets prepended once. - let prefix = build_prefix([&Bytes::from_static(&[0x67, 0x42])]); - let payload = &[ - 0, 0, 0, 2, 0x09, 0x10, // AUD - 0, 0, 0, 2, 0x65, 0x88, // IDR slice - ]; - let out = from_length_prefixed(payload, 4, Some(&prefix)).unwrap(); + fn from_length_prefixed_keeps_aud_first() { + for (params, aud, slice) in [ + ( + vec![vec![0x67, 0x42], vec![0x68, 0xce]], + vec![0x09, 0x10], + vec![0x65, 0x88], + ), + ( + vec![vec![0x40, 1], vec![0x42, 1], vec![0x44, 1]], + vec![0x46, 1, 0x10], + vec![0x26, 1, 0x88], + ), + ] { + let params: Vec = params.into_iter().map(Bytes::from).collect(); + let prefix = build_prefix(¶ms); + for length_size in 1..=4 { + for leading in [None, Some(aud.as_slice())] { + let mut payload = Vec::new(); + for nal in leading.into_iter().chain([slice.as_slice()]) { + payload.extend_from_slice(&(nal.len() as u32).to_be_bytes()[4 - length_size..]); + payload.extend_from_slice(nal); + } + let mut expected = Vec::new(); + if let Some(aud) = leading { + expected.extend_from_slice(&START_CODE); + expected.extend_from_slice(aud); + } + expected.extend_from_slice(&prefix); + expected.extend_from_slice(&START_CODE); + expected.extend_from_slice(&slice); + let out = from_length_prefixed(&payload, length_size, Some(&prefix)).unwrap(); + assert_eq!(out.as_ref(), expected.as_slice()); + let mut without_prefix = Vec::new(); + for nal in leading.into_iter().chain([slice.as_slice()]) { + without_prefix.extend_from_slice(&START_CODE); + without_prefix.extend_from_slice(nal); + } + assert_eq!( + from_length_prefixed(&payload, length_size, None).unwrap(), + without_prefix + ); + } + } + } + } - // Single prefix followed by both NALs in Annex-B order. - let mut expected = Vec::new(); - expected.extend_from_slice(&prefix); - expected.extend_from_slice(&[0, 0, 0, 1, 0x09, 0x10]); // AUD - expected.extend_from_slice(&[0, 0, 0, 1, 0x65, 0x88]); // IDR - assert_eq!(out.as_ref(), expected.as_slice()); + #[test] + fn from_length_prefixed_distinguishes_codec_headers() { + // H.265 TSA_N with nuh_layer_id 32 has the same first byte as an H.264 AUD. + let prefix = build_prefix([&Bytes::from_static(&[0x40, 1])]); + let payload = &[0, 0, 0, 3, 0x09, 1, 0x80]; + let mut expected = prefix.to_vec(); + expected.extend_from_slice(&[0, 0, 0, 1, 0x09, 1, 0x80]); + assert_eq!(from_length_prefixed(payload, 4, Some(&prefix)).unwrap(), expected); } #[test] diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index c3892e71c46..20e7ef70fc3 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -3619,9 +3619,9 @@ fn ac3_sync_prefix(bytes: &[u8]) -> bool { } /// Build the Annex-B elementary-stream payload for one video frame: rewrite the -/// length-prefixed NALs to start-code-delimited NALs, prepending the parameter -/// sets (SPS/PPS, plus VPS for H.265) from the avcC/hvcC on keyframes so a -/// receiver can tune in mid-stream. +/// length-prefixed NALs to start-code-delimited NALs, injecting the parameter +/// sets (SPS/PPS, plus VPS for H.265) from the avcC/hvcC after a leading AUD on +/// keyframes so a receiver can tune in mid-stream. fn video_es_payload(stream_type: StreamType, description: Option<&Bytes>, frame: &Frame) -> anyhow::Result> { let description = description.context("video codec config (avcC/hvcC) not resolved")?; let (length_size, params) = match stream_type { @@ -3631,13 +3631,21 @@ fn video_es_payload(stream_type: StreamType, description: Option<&Bytes>, frame: }; let mut out = Vec::with_capacity(frame.payload.len() + 64); + let mut nals = annexb::length_prefixed_nals(&frame.payload, length_size)?; if frame.keyframe { + if let Some(aud) = nals.take_delimiter(params.first().and_then(|nal| nal.first()).copied())? { + out.extend_from_slice(&annexb::START_CODE); + out.extend_from_slice(aud); + } for nal in ¶ms { out.extend_from_slice(&annexb::START_CODE); out.extend_from_slice(nal); } } - annexb::length_prefixed_to_annexb(&frame.payload, length_size, &mut out)?; + for nal in nals { + out.extend_from_slice(&annexb::START_CODE); + out.extend_from_slice(nal?); + } Ok(out) } @@ -3868,6 +3876,68 @@ mod tests { Timestamp::from_millis(value).unwrap() } + #[test] + fn video_es_payload_keeps_aud_first() { + for (stream_type, params, aud, slice) in [ + ( + super::StreamType::H264, + vec![vec![0x67, 0x42, 0, 0x1f], vec![0x68, 0xce]], + vec![0x09, 0x10], + vec![0x65, 0x88], + ), + ( + super::StreamType::H265, + vec![vec![0x40, 1], vec![0x42, 1], vec![0x44, 1]], + vec![0x46, 1, 0x10], + vec![0x26, 1, 0x88], + ), + ] { + let params: Vec = params.into_iter().map(bytes::Bytes::from).collect(); + let description = if stream_type == super::StreamType::H264 { + crate::codec::h264::build_avcc(¶ms[..1], ¶ms[1..]).unwrap() + } else { + let mut hvcc = vec![0; 23]; + hvcc[21] = 3; + hvcc[22] = 3; + for (kind, nal) in (32..=34).zip(¶ms) { + hvcc.extend_from_slice(&[kind, 0, 1]); + hvcc.extend_from_slice(&(nal.len() as u16).to_be_bytes()); + hvcc.extend_from_slice(nal); + } + hvcc.into() + }; + for keyframe in [false, true] { + for leading in [None, Some(aud.as_slice())] { + let mut payload = Vec::new(); + for nal in leading.into_iter().chain([slice.as_slice()]) { + payload.extend_from_slice(&(nal.len() as u32).to_be_bytes()); + payload.extend_from_slice(nal); + } + let frame = super::Frame { + timestamp: ms(0), + payload: payload.into(), + keyframe, + duration: None, + }; + let mut expected = Vec::new(); + if let Some(aud) = leading { + expected.extend_from_slice(&super::annexb::START_CODE); + expected.extend_from_slice(aud); + } + if keyframe { + expected.extend_from_slice(&super::annexb::build_prefix(¶ms)); + } + expected.extend_from_slice(&super::annexb::START_CODE); + expected.extend_from_slice(&slice); + assert_eq!( + super::video_es_payload(stream_type, Some(&description), &frame).unwrap(), + expected + ); + } + } + } + } + /// A PES that completes the AC-3 sync frame the last one began holds a frame decoding /// ahead of its own PTS. Each sync frame reaches the jitter buffer at its own decode time, /// so the buffer orders that frame against the other tracks on its own time, not the PES's. From adaced18ac4a2d1d7b01c30cbfca001d0c194674 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 19 Oct 2026 17:13:31 -0700 Subject: [PATCH 3/3] refactor(mux): pass the codec to take_delimiter The TS writer already knows its codec from the stream type, so only from_length_prefixed, whose signature carries no codec, infers it from the injected parameter-set header. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/codec/annexb.rs | 32 +++++++++++++++++---------- rs/moq-mux/src/container/ts/export.rs | 9 ++++---- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/rs/moq-mux/src/codec/annexb.rs b/rs/moq-mux/src/codec/annexb.rs index bee47206b53..4f51fe6f592 100644 --- a/rs/moq-mux/src/codec/annexb.rs +++ b/rs/moq-mux/src/codec/annexb.rs @@ -57,17 +57,11 @@ pub(crate) struct LengthPrefixedNals<'a> { } impl<'a> LengthPrefixedNals<'a> { - /// Take a leading AUD, identifying the codec from an injected parameter-set header. - pub(crate) fn take_delimiter(&mut self, parameter_header: Option) -> Result> { - // NAL headers overlap between codecs: an H.265 slice can look like an H.264 AUD. - let codec = parameter_header.and_then(|header| match (header & 0x1f, (header >> 1) & 0x3f) { - (7 | 8, _) => Some(InBandCodec::H264), - (_, 32..=34) => Some(InBandCodec::H265), - _ => None, - }); + /// Take a leading access unit delimiter, which must stay ahead of injected parameter sets. + pub(crate) fn take_delimiter(&mut self, codec: InBandCodec) -> Result> { if let Some(nal) = self.clone().next().transpose()? && !nal.is_empty() - && codec.is_some_and(|codec| codec.nal_type(nal) == codec.delimiter()) + && codec.nal_type(nal) == codec.delimiter() { return self.next().transpose(); } @@ -105,12 +99,14 @@ pub fn from_length_prefixed(payload: &[u8], length_size: usize, prefix: Option<& let mut out = BytesMut::with_capacity(payload.len() + prefix.map(|p| p.len()).unwrap_or(0) + 16); let mut nals = length_prefixed_nals(payload, length_size)?; if let Some(prefix) = prefix { - let header = after_start_code(prefix) + let codec = after_start_code(prefix) .ok() .flatten() .and_then(|start| prefix.get(start)) - .copied(); - if let Some(aud) = nals.take_delimiter(header)? { + .and_then(|&header| InBandCodec::of_parameter_set(header)); + if let Some(codec) = codec + && let Some(aud) = nals.take_delimiter(codec)? + { out.extend_from_slice(&START_CODE); out.extend_from_slice(aud); } @@ -167,6 +163,18 @@ impl InBandCodec { } } + /// The codec whose parameter-set NAL begins with `header`. + /// + /// NAL headers overlap between codecs (an H.265 slice can look like an H.264 AUD), so + /// only a parameter set identifies one. + fn of_parameter_set(header: u8) -> Option { + match (header & 0x1f, (header >> 1) & 0x3f) { + (7 | 8, _) => Some(Self::H264), + (_, 32..=34) => Some(Self::H265), + _ => None, + } + } + fn delimiter(self) -> u8 { match self { Self::H264 => 9, diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 0cbebdb839f..78b107c3dd1 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -43,6 +43,7 @@ use moq_net::Timestamp; use super::catalog::Catalog as _; use crate::catalog::Stream; use crate::catalog::hang::Catalog; +use crate::codec::annexb::InBandCodec; use crate::codec::video::{Hrd, Reorder}; use crate::codec::{aac, annexb, opus}; use crate::container::decode::{DecodeClock, MAX_REORDER}; @@ -3214,16 +3215,16 @@ fn ac3_sync_prefix(bytes: &[u8]) -> bool { /// keyframes so a receiver can tune in mid-stream. fn video_es_payload(stream_type: StreamType, description: Option<&Bytes>, frame: &Frame) -> anyhow::Result> { let description = description.context("video codec config (avcC/hvcC) not resolved")?; - let (length_size, params) = match stream_type { - StreamType::H264 => crate::codec::h264::avcc_params(description)?, - StreamType::H265 => crate::codec::h265::hvcc_params(description)?, + let (codec, (length_size, params)) = match stream_type { + StreamType::H264 => (InBandCodec::H264, crate::codec::h264::avcc_params(description)?), + StreamType::H265 => (InBandCodec::H265, crate::codec::h265::hvcc_params(description)?), other => anyhow::bail!("unsupported TS video stream type {other:?}"), }; let mut out = Vec::with_capacity(frame.payload.len() + 64); let mut nals = annexb::length_prefixed_nals(&frame.payload, length_size)?; if frame.keyframe { - if let Some(aud) = nals.take_delimiter(params.first().and_then(|nal| nal.first()).copied())? { + if let Some(aud) = nals.take_delimiter(codec)? { out.extend_from_slice(&annexb::START_CODE); out.extend_from_slice(aud); }