Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion quest/m1/ts-reliability/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,6 @@ remain with their current quests.
- [Plan: semantic fidelity](/quest/m1/ts-reliability/fidelity.md) - explicit profiles and PMT lifecycle contracts produce implementation quests
- [Section placement](/quest/m1/ts-reliability/section-placement.md) - sections share the PCR placement clock instead of the video PTS
- [Sequence holes get a time budget](/quest/m1/ts-reliability/hole-budget.md) - a hole gives up on max(wall, pts), so a break marker behind it on a paused track is delivered
- [AUD first](/quest/m1/ts-reliability/aud-first.md) - a keyframe's access unit delimiter stays ahead of the re-injected parameter sets
- [TS source front end](/quest/m1/ts-reliability/source-front-end.md) - cli.md says to pipe a TS source's bytes, and the first PMT logs what each stream became
- [Loss and recovery tests](/quest/m1/ts-reliability/recovery.md) - actual MoQ omissions exercise reacquisition and unaffected tracks

Expand Down
28 changes: 0 additions & 28 deletions quest/m1/ts-reliability/aud-first.md

This file was deleted.

117 changes: 96 additions & 21 deletions rs/moq-mux/src/codec/annexb.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,25 @@ pub(crate) fn length_prefixed_nals(data: &[u8], length_size: usize) -> Result<Le
}

/// The iterator [`length_prefixed_nals`] returns.
#[derive(Clone)]
pub(crate) struct LengthPrefixedNals<'a> {
data: &'a [u8],
length_size: usize,
}

impl<'a> LengthPrefixedNals<'a> {
/// Take a leading access unit delimiter, which must stay ahead of injected parameter sets.
pub(crate) fn take_delimiter(&mut self, codec: InBandCodec) -> Result<Option<&'a [u8]>> {
if let Some(nal) = self.clone().next().transpose()?
&& !nal.is_empty()
&& codec.nal_type(nal) == codec.delimiter()
{
return self.next().transpose();
}
Ok(None)
}
}

impl<'a> Iterator for LengthPrefixedNals<'a> {
type Item = Result<&'a [u8]>;

Expand All @@ -79,19 +93,29 @@ impl<'a> Iterator for LengthPrefixedNals<'a> {
}

/// Convert a length-prefixed NALU payload (avc1 / hvc1 wire shape) to Annex-B,
/// optionally prepending `prefix` bytes (typically VPS/SPS/PPS NAL units already
/// in Annex-B form, for keyframe parameter-set injection).
/// injecting `prefix` bytes (typically VPS/SPS/PPS NAL units already in Annex-B
/// form) after a leading access unit delimiter, or before the other NAL units.
pub fn from_length_prefixed(payload: &[u8], length_size: usize, prefix: Option<&[u8]>) -> Result<Bytes> {
let mut out = BytesMut::with_capacity(payload.len() + prefix.map(|p| p.len()).unwrap_or(0) + 16);
if let Some(p) = prefix {
out.extend_from_slice(p);
let mut nals = length_prefixed_nals(payload, length_size)?;
if let Some(prefix) = prefix {
let codec = after_start_code(prefix)
.ok()
.flatten()
.and_then(|start| prefix.get(start))
.and_then(|&header| InBandCodec::of_parameter_set(header));
if let Some(codec) = codec
&& let Some(aud) = nals.take_delimiter(codec)?
{
out.extend_from_slice(&START_CODE);
out.extend_from_slice(aud);
}
out.extend_from_slice(prefix);
}

for nal in length_prefixed_nals(payload, length_size)? {
for nal in nals {
out.extend_from_slice(&START_CODE);
out.extend_from_slice(nal?);
}

Ok(out.freeze())
}

Expand Down Expand Up @@ -139,6 +163,18 @@ impl InBandCodec {
}
}

/// The codec whose parameter-set NAL begins with `header`.
///
/// NAL headers overlap between codecs (an H.265 slice can look like an H.264 AUD), so
/// only a parameter set identifies one.
fn of_parameter_set(header: u8) -> Option<Self> {
match (header & 0x1f, (header >> 1) & 0x3f) {
(7 | 8, _) => Some(Self::H264),
(_, 32..=34) => Some(Self::H265),
_ => None,
}
}

fn delimiter(self) -> u8 {
match self {
Self::H264 => 9,
Expand Down Expand Up @@ -412,21 +448,60 @@ mod tests {
}

#[test]
fn from_length_prefixed_multiple_nals_with_prefix() {
// Two NALs in one frame: AUD then IDR slice. Prefix gets prepended once.
let prefix = build_prefix([&Bytes::from_static(&[0x67, 0x42])]);
let payload = &[
0, 0, 0, 2, 0x09, 0x10, // AUD
0, 0, 0, 2, 0x65, 0x88, // IDR slice
];
let out = from_length_prefixed(payload, 4, Some(&prefix)).unwrap();
fn from_length_prefixed_keeps_aud_first() {
for (params, aud, slice) in [
(
vec![vec![0x67, 0x42], vec![0x68, 0xce]],
vec![0x09, 0x10],
vec![0x65, 0x88],
),
(
vec![vec![0x40, 1], vec![0x42, 1], vec![0x44, 1]],
vec![0x46, 1, 0x10],
vec![0x26, 1, 0x88],
),
] {
let params: Vec<Bytes> = params.into_iter().map(Bytes::from).collect();
let prefix = build_prefix(&params);
for length_size in 1..=4 {
for leading in [None, Some(aud.as_slice())] {
let mut payload = Vec::new();
for nal in leading.into_iter().chain([slice.as_slice()]) {
payload.extend_from_slice(&(nal.len() as u32).to_be_bytes()[4 - length_size..]);
payload.extend_from_slice(nal);
}
let mut expected = Vec::new();
if let Some(aud) = leading {
expected.extend_from_slice(&START_CODE);
expected.extend_from_slice(aud);
}
expected.extend_from_slice(&prefix);
expected.extend_from_slice(&START_CODE);
expected.extend_from_slice(&slice);
let out = from_length_prefixed(&payload, length_size, Some(&prefix)).unwrap();
assert_eq!(out.as_ref(), expected.as_slice());
let mut without_prefix = Vec::new();
for nal in leading.into_iter().chain([slice.as_slice()]) {
without_prefix.extend_from_slice(&START_CODE);
without_prefix.extend_from_slice(nal);
}
assert_eq!(
from_length_prefixed(&payload, length_size, None).unwrap(),
without_prefix
);
}
}
}
}

// Single prefix followed by both NALs in Annex-B order.
let mut expected = Vec::new();
expected.extend_from_slice(&prefix);
expected.extend_from_slice(&[0, 0, 0, 1, 0x09, 0x10]); // AUD
expected.extend_from_slice(&[0, 0, 0, 1, 0x65, 0x88]); // IDR
assert_eq!(out.as_ref(), expected.as_slice());
#[test]
fn from_length_prefixed_distinguishes_codec_headers() {
// H.265 TSA_N with nuh_layer_id 32 has the same first byte as an H.264 AUD.
let prefix = build_prefix([&Bytes::from_static(&[0x40, 1])]);
let payload = &[0, 0, 0, 3, 0x09, 1, 0x80];
let mut expected = prefix.to_vec();
expected.extend_from_slice(&[0, 0, 0, 1, 0x09, 1, 0x80]);
assert_eq!(from_length_prefixed(payload, 4, Some(&prefix)).unwrap(), expected);
}

#[test]
Expand Down
85 changes: 78 additions & 7 deletions rs/moq-mux/src/container/ts/export.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ use moq_net::Timestamp;
use super::catalog::Catalog as _;
use crate::catalog::Stream;
use crate::catalog::hang::Catalog;
use crate::codec::annexb::InBandCodec;
use crate::codec::video::{Hrd, Reorder};
use crate::codec::{aac, annexb, opus};
use crate::container::decode::{DecodeClock, MAX_REORDER};
Expand Down Expand Up @@ -3209,25 +3210,33 @@ fn ac3_sync_prefix(bytes: &[u8]) -> bool {
}

/// Build the Annex-B elementary-stream payload for one video frame: rewrite the
/// length-prefixed NALs to start-code-delimited NALs, prepending the parameter
/// sets (SPS/PPS, plus VPS for H.265) from the avcC/hvcC on keyframes so a
/// receiver can tune in mid-stream.
/// length-prefixed NALs to start-code-delimited NALs, injecting the parameter
/// sets (SPS/PPS, plus VPS for H.265) from the avcC/hvcC after a leading AUD on
/// keyframes so a receiver can tune in mid-stream.
fn video_es_payload(stream_type: StreamType, description: Option<&Bytes>, frame: &Frame) -> anyhow::Result<Vec<u8>> {
let description = description.context("video codec config (avcC/hvcC) not resolved")?;
let (length_size, params) = match stream_type {
StreamType::H264 => crate::codec::h264::avcc_params(description)?,
StreamType::H265 => crate::codec::h265::hvcc_params(description)?,
let (codec, (length_size, params)) = match stream_type {
StreamType::H264 => (InBandCodec::H264, crate::codec::h264::avcc_params(description)?),
StreamType::H265 => (InBandCodec::H265, crate::codec::h265::hvcc_params(description)?),
other => anyhow::bail!("unsupported TS video stream type {other:?}"),
};

let mut out = Vec::with_capacity(frame.payload.len() + 64);
let mut nals = annexb::length_prefixed_nals(&frame.payload, length_size)?;
if frame.keyframe {
if let Some(aud) = nals.take_delimiter(codec)? {
out.extend_from_slice(&annexb::START_CODE);
out.extend_from_slice(aud);
}
for nal in &params {
out.extend_from_slice(&annexb::START_CODE);
out.extend_from_slice(nal);
}
}
annexb::length_prefixed_to_annexb(&frame.payload, length_size, &mut out)?;
for nal in nals {
out.extend_from_slice(&annexb::START_CODE);
out.extend_from_slice(nal?);
}
Ok(out)
}

Expand Down Expand Up @@ -3458,6 +3467,68 @@ mod tests {
Timestamp::from_millis(value).unwrap()
}

#[test]
fn video_es_payload_keeps_aud_first() {
for (stream_type, params, aud, slice) in [
(
super::StreamType::H264,
vec![vec![0x67, 0x42, 0, 0x1f], vec![0x68, 0xce]],
vec![0x09, 0x10],
vec![0x65, 0x88],
),
(
super::StreamType::H265,
vec![vec![0x40, 1], vec![0x42, 1], vec![0x44, 1]],
vec![0x46, 1, 0x10],
vec![0x26, 1, 0x88],
),
] {
let params: Vec<bytes::Bytes> = params.into_iter().map(bytes::Bytes::from).collect();
let description = if stream_type == super::StreamType::H264 {
crate::codec::h264::build_avcc(&params[..1], &params[1..]).unwrap()
} else {
let mut hvcc = vec![0; 23];
hvcc[21] = 3;
hvcc[22] = 3;
for (kind, nal) in (32..=34).zip(&params) {
hvcc.extend_from_slice(&[kind, 0, 1]);
hvcc.extend_from_slice(&(nal.len() as u16).to_be_bytes());
hvcc.extend_from_slice(nal);
}
hvcc.into()
};
for keyframe in [false, true] {
for leading in [None, Some(aud.as_slice())] {
let mut payload = Vec::new();
for nal in leading.into_iter().chain([slice.as_slice()]) {
payload.extend_from_slice(&(nal.len() as u32).to_be_bytes());
payload.extend_from_slice(nal);
}
let frame = super::Frame {
timestamp: ms(0),
payload: payload.into(),
keyframe,
duration: None,
};
let mut expected = Vec::new();
if let Some(aud) = leading {
expected.extend_from_slice(&super::annexb::START_CODE);
expected.extend_from_slice(aud);
}
if keyframe {
expected.extend_from_slice(&super::annexb::build_prefix(&params));
}
expected.extend_from_slice(&super::annexb::START_CODE);
expected.extend_from_slice(&slice);
assert_eq!(
super::video_es_payload(stream_type, Some(&description), &frame).unwrap(),
expected
);
}
}
}
}

/// A six-block E-AC-3 sync frame of 512 bytes: `stream` is byte 2's top bits (strmtyp,
/// substreamid) and `rate` byte 4 (fscod, numblkscod, acmod, lfeon).
fn eac3(stream: u8, rate: u8) -> Vec<u8> {
Expand Down
Loading