diff --git a/doc/bin/cli.md b/doc/bin/cli.md index edd7ae84772..3af88af2ade 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -139,6 +139,12 @@ and LTP profiles. HE-AAC and HE-AACv2 go out as their AAC-LC core, and decoders find the SBR and PS in band, as ffmpeg's ADTS output does. A track whose profile or channel layout ADTS cannot label is refused rather than mislabeled. +An AAC layout without a channelConfiguration, such as quad, rides in a program +config element. `export ts` repeats it in the first frame after each PAT/PMT, so a +receiver tuning in mid-stream can decode. ffmpeg writes it in the first frame +only, so `import ts` joining an ffmpeg source mid-stream publishes the rest of +the program without that AAC track, and adds the track if an element arrives. + A constant-rate MPEG-TS source records its multiplex rate in the catalog (`mpegts.muxRate`, measured off the PCR clock, null stuffing included), and `export ts` pads its output with null packets back to that rate so an IRD or diff --git a/quest/m1/README.md b/quest/m1/README.md index b4ac1f6d833..d5a26faf9ae 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -108,7 +108,6 @@ blocks. The quests that gated m0 lines moved under them. - [AAC encode refusals](/quest/m1/aac-encode-refusals.md) - `Config::encode` refuses channel counts it cannot name instead of writing stereo - [AAC parse truncated SBR](/quest/m1/aac-parse-truncated-sbr.md) - `Config::parse` refuses SBR and PS configs cut off before their core - [GStreamer surround Opus](/quest/m1/gst-opus-surround.md) - the moq-gst sink publishes 3 to 8 channel Opus with the OpusHead its caps describe -- [TS AAC PCE joins](/quest/m1/ts-aac-pce-join.md) - PCE-described AAC over TS plays after a mid-stream join or resume, and a missing PCE silences only its track - [FFI frame duration default](/quest/m1/ffi-frame-duration-default.md) - the binding audio encoder takes the codec's own frame by default, so `aac()` needs no explicit 0 - [mp4-atom dOps mapping](/quest/m1/mp4-atom-dops-mapping.md) - a released mp4-atom reads and writes any `dOps` channel mapping family and table - [CMAF surround Opus](/quest/m1/cmaf-opus-surround.md) - fMP4 import and export carry an Opus channel mapping table diff --git a/quest/m1/ts-aac-pce-join.md b/quest/m1/ts-aac-pce-join.md deleted file mode 100644 index c519dee1bb4..00000000000 --- a/quest/m1/ts-aac-pce-join.md +++ /dev/null @@ -1,36 +0,0 @@ -# [S] TS AAC program config survives joins and resumes - -## Goal - -AAC whose layout lives in a program config element (ADTS `channel_config` 0) -plays over MPEG-TS for a receiver that joins mid-stream or after a resume. A -missing PCE silences that one track instead of failing the whole program. - -## Plan - -Found in review of the audio-codecs line (#4081), 2026-09-30. - -- Import: since #4733 a missing PCE no longer ends the import; it is a - Damaged unit (`unit_error` in `rs/moq-mux/src/container/ts/import.rs`). But - the AAC track still holds its catalog reservation until a PCE arrives, and - the catalog is withheld until every reservation drops - (`rs/moq-mux/src/catalog/producer.rs`), so a late join to an ffmpeg source - publishes nothing, video included. Decided in the 2026-10-05 audit: a - PCE-less AAC track releases its reservation, the catalog publishes without - it, and the track is added when a PCE arrives. Rejected: keeping the - reservation and withholding the program. ffmpeg's ADTS muxer writes the PCE - only in the first frame, so a late join of an ffmpeg source plays without - that AAC track; say so in the docs. -- Export: `mux` takes the PCE on the first frame only - (`rs/moq-mux/src/container/ts/export.rs`), so `resume()`/`rewind()` and any - mid-stream receiver never see it. Repeat it, either on every frame or with - each PAT/PMT, and pick whichever ffmpeg and the import side accept. -- Regression: an export resumed before its first span re-emits the PCE, and an - import that starts on a PCE-less frame publishes its catalog with the other - tracks at once, then adds the AAC track once a PCE arrives. - -Public API: none. Wire: none; TS output gains repeated PCEs. - -## Related - -- [TS Opus channel codes](/quest/m1/ts-opus-channel-codes.md) - the same per-track refusal principle for Opus descriptors diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 50fa3979d38..86ef9a29870 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -402,9 +402,9 @@ enum Kind { /// Video carries its TS stream type (H.264 = 0x1B, H.265 = 0x24). Video(StreamType), /// AAC, framed as ADTS. A `channel_config` of 0 defers the layout to a program config - /// element, which leads the next raw data block written and is then taken. A catalog update - /// rebuilds the kind and so repeats it once, which a decoder tuning in mid-stream welcomes. - Aac(aac::InBand), + /// element, which leads the first raw data block after each PAT/PMT, so a receiver that tunes + /// in at the tables can decode from there. `repeat` marks that the next frame carries it. + Aac { config: aac::InBand, repeat: bool }, /// Opus (private stream_type 0x06). Each frame is one Opus packet, prefixed with /// the Opus-in-TS access-unit control header and announced with the 'Opus' /// registration plus DVB extension descriptor. @@ -1486,7 +1486,7 @@ impl Export { tracks.iter().find(|t| { matches!( t.kind, - Kind::Aac(_) | Kind::Opus { .. } | Kind::Mp2 { .. } | Kind::Ac3 | Kind::Eac3 + Kind::Aac { .. } | Kind::Opus { .. } | Kind::Mp2 { .. } | Kind::Ac3 | Kind::Eac3 ) }) }) @@ -1498,7 +1498,7 @@ impl Export { .map(|t| { let stream_type = match &t.kind { Kind::Video(stream_type) => *stream_type, - Kind::Aac(_) => StreamType::AdtsAac, + Kind::Aac { .. } => StreamType::AdtsAac, // Opus rides private-data PES; the registration + extension descriptors // below tell the demuxer it's Opus. Kind::Opus { .. } => StreamType::from_u8(0x06).map_err(anyhow::Error::msg)?, @@ -1653,13 +1653,22 @@ impl Export { if self.span_counters.is_none() { self.span_counters = Some(self.counters.clone()); } + let is_video = matches!(self.tracks.get(name).context("missing track")?.kind, Kind::Video(_)); + // Refresh PSI at keyframes or after the interval lapses. + let psi = (is_video && frame.keyframe) || due(frame.timestamp, self.last_psi, PSI_INTERVAL); + if psi { + for track in self.tracks.values_mut() { + if let Kind::Aac { repeat, .. } = &mut track.kind { + *repeat = true; + } + } + } let track = self.tracks.get_mut(name).context("missing track")?; let pid = track.pid; let kind = track.kind.clone(); - if let Kind::Aac(aac) = &mut track.kind { - aac.program_config.take(); + if let Kind::Aac { repeat, .. } = &mut track.kind { + *repeat = false; } - let is_video = matches!(kind, Kind::Video(_)); let timestamp = frame.timestamp; let keyframe = frame.keyframe; @@ -1668,10 +1677,15 @@ impl Export { // verbatim streams carry no PES payload; the section is written separately below. let es_payload = match &kind { Kind::Video(stream_type) => Some(video_es_payload(*stream_type, track.source.description(), &frame)?), - Kind::Aac(aac) => { - let pce = aac.program_config.as_deref().unwrap_or_default(); + Kind::Aac { config, repeat } => { + let pce = if *repeat { + config.program_config.as_deref().unwrap_or_default() + } else { + &[] + }; let raw_len = pce.len() + frame.payload.len(); - let header = adts::write_header(aac.object_type, aac.sample_rate, aac.channel_config, raw_len)?; + let header = + adts::write_header(config.object_type, config.sample_rate, config.channel_config, raw_len)?; let mut framed = Vec::with_capacity(header.len() + raw_len); framed.extend_from_slice(&header); framed.extend_from_slice(pce); @@ -1705,8 +1719,7 @@ impl Export { let mut out = Vec::with_capacity(TsPacket::SIZE); - // Refresh PSI at keyframes or after the interval lapses. - if (is_video && frame.keyframe) || due(frame.timestamp, self.last_psi, PSI_INTERVAL) { + if psi { let psi = self.psi.as_ref().context("PSI not built")?; let pmt_pid = psi.pmt_pid; let pat = TsPayload::Pat(psi.pat.clone()); @@ -2435,7 +2448,7 @@ fn audio_kind(config: &AudioConfig, name: &str) -> anyhow::Result { AudioCodec::AAC(codec) => { // The description is exact, and names the LC core under explicit SBR or PS. Without // one, the catalog is all there is. - Ok(Kind::Aac(match &config.description { + let in_band = match &config.description { Some(asc) => aac::in_band(asc)?, None => aac::InBand { object_type: codec.profile, @@ -2443,7 +2456,13 @@ fn audio_kind(config: &AudioConfig, name: &str) -> anyhow::Result { channel_config: adts::channel_config_from_count(config.channel_count)?, program_config: None, }, - })) + }; + // A rebuilt kind repeats the element too, so a catalog update between a PAT/PMT and + // this track's next frame cannot drop it. + Ok(Kind::Aac { + config: in_band, + repeat: true, + }) } AudioCodec::Mp2 => Ok(Kind::Mp2 { sample_rate: config.sample_rate, diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 5f1dd5bfe65..4920ae1ea36 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -1481,7 +1481,121 @@ async fn aac_program_config_roundtrip() { assert_eq!(audio2.channel_count, 4); assert_eq!(audio2.description, audio.description); let roundtripped = read_frames(&consumer2, name2, Kind::Audio).await; - assert_eq!(roundtripped, ingested, "the element is written once, not per frame"); + assert_eq!(roundtripped, ingested, "the element leaves the frames on import"); +} + +/// For each AAC PES, whether it follows a PAT and whether a program config element leads its +/// first raw data block. +fn aac_program_configs(ts: &[u8]) -> Vec<(bool, bool)> { + let mut reader = TsPacketReader::new(Cursor::new(ts)); + let mut out = Vec::new(); + let mut after_pat = false; + while let Some(packet) = reader.read_ts_packet().unwrap() { + match packet.payload { + Some(TsPayload::Pat(_)) => after_pat = true, + Some(TsPayload::PesStart(pes)) => { + let header = super::adts::Header::parse(&pes.data).unwrap(); + assert_eq!(header.channel_config, 0); + // ID_PCE in the first three bits of the raw data block. + out.push((std::mem::take(&mut after_pat), pes.data[header.header_len] >> 5 == 5)); + } + _ => {} + } + } + out +} + +/// A receiver tunes in at a PAT/PMT, so the program config element follows each one rather +/// than riding the first frame only, including after a marker restarts the program clock. The +/// output cut at a later PAT imports as the same quad track. +#[tokio::test(start_paused = true)] +async fn aac_program_config_follows_each_table() { + let mut quad = vec![0x11, 0x80, 0x04, 0xC4, 0x04, 0x00, 0x21, 0x10, 0x0C]; + quad.extend_from_slice(b"Lavc63.1.101"); + let payload = Bytes::from_static(&[0x20; 10]); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let track = broadcast + .create_track("a.aac", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut cfg = AudioConfig::new(AAC { profile: 2 }, 48_000, 4); + cfg.container = Container::Legacy; + cfg.description = Some(Bytes::from(quad.clone())); + catalog + .modify() + .unwrap() + .audio + .renditions + .insert("a.aac".to_string(), cfg); + let mut producer = Producer::new(track, HangContainer::Legacy(Kind::Audio)); + let mut export = Export::new(crate::source::announced(&consumer)) + .await + .unwrap() + .with_max_age(RECORDING_MAX_AGE); + + let write = |producer: &mut Producer, ms: std::ops::Range| { + for ms in ms.step_by(100) { + producer + .write(Frame { + timestamp: Timestamp::from_millis(ms).unwrap(), + duration: None, + payload: payload.clone(), + keyframe: true, + }) + .unwrap(); + producer.cut(None).unwrap(); + } + }; + write(&mut producer, 0..2_000); + producer.discontinuity().unwrap(); + write(&mut producer, 2_000..3_000); + producer.finish().unwrap(); + let frames = drain_frames(&mut export).await; + let ts: Vec = frames.iter().flat_map(|f| f.payload.iter().copied()).collect(); + assert_packet_aligned(&ts); + + let units = aac_program_configs(&ts); + assert_eq!(units.len(), 30); + for (i, (after_pat, pce)) in units.iter().enumerate() { + assert_eq!( + after_pat, pce, + "unit {i}: the element rides with the tables, and only there" + ); + } + assert!(units[20].1, "the restarted clock re-sends the element"); + + // Join at the second PAT. + let pats: Vec = ts + .chunks(188) + .enumerate() + .filter(|(_, p)| p[1] & 0x1f == 0 && p[2] == 0) + .map(|(i, _)| i * 188) + .collect(); + assert!(pats.len() > 2, "PAT on its cadence"); + let mut joined = moq_net::broadcast::Info::new().produce(); + let joined_consumer = joined.consume(); + let joined_catalog = crate::catalog::Producer::new(&mut joined, crate::catalog::Config::default()).unwrap(); + let mut import = crate::container::ts::Import::new(joined, joined_catalog.reserve()); + import.decode(&BytesMut::from(&ts[pats[1]..])).unwrap(); + import.finish().unwrap(); + + let snapshot = joined_catalog.snapshot(); + let (name, audio) = snapshot + .audio + .renditions + .iter() + .next() + .expect("the late join lost the AAC track"); + assert_eq!(audio.channel_count, 4); + assert_eq!(audio.description.as_deref(), Some(quad.as_slice())); + let imported = read_frames(&joined_consumer, name, Kind::Audio).await; + assert!(!imported.is_empty()); + assert!( + imported.iter().all(|frame| frame[..] == payload[..]), + "the repeated element leaves every frame" + ); } /// GStreamer 1.28 `fdkaacenc` output, 48 kHz stereo, remuxed to FLV by ffmpeg 9.0.1. Its diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 3f574b52032..0aa70ef7c7b 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -498,8 +498,10 @@ impl Import { // framing and syncword, so it falls through to the ignored arm below. Some(StreamType::AdtsAac) => Stream::Aac(Box::new(AacStream { import: None, + asc: bytes::Bytes::new(), broadcast: self.broadcast.clone(), reserved: Some(self.reserve()), + catalog: self.catalog.clone(), container: self.container.clone(), unwrap: PtsUnwrap::default(), tail: Vec::new(), @@ -2168,11 +2170,16 @@ impl From<&legacy::Descriptor> for SyncWord { /// deferred until the first frame arrives. struct AacStream { import: Option, + /// The AudioSpecificConfig `import` was built with. A program config element in a later frame + /// must rebuild it exactly. + asc: bytes::Bytes, broadcast: moq_net::broadcast::Producer, /// Reservation held from the PMT until the first frame builds the importer, so the catalog stays /// withheld until this deferred rendition resolves (config comes from the first ADTS header). - /// Consumed when `import` is built. + /// Consumed when `import` is built, or released by a frame that cannot build it. reserved: Option>, + /// Reserves the rendition anew when `import` is built after `reserved` was released. + catalog: crate::catalog::Producer, /// The container this importer publishes decoded renditions with. container: hang::catalog::Container, unwrap: PtsUnwrap, @@ -2314,7 +2321,24 @@ impl AacStream { let mut block = &data[offset + header.header_len..end]; let import = match &mut self.import { - Some(import) => import, + Some(import) => { + // The description already carries the layout, so a repeated program config + // element (the TS export writes one after each PAT/PMT) leaves the frame too. + if header.channel_config == 0 { + let mut rest = block; + match aac::in_band_config(header.object_type, header.sample_rate, 0, &mut rest) { + Ok(asc) if asc == self.asc => block = rest, + Ok(_) => { + return Err( + Damaged(anyhow::anyhow!("AAC program config element changed mid-stream")).into(), + ); + } + Err(aac::Error::ProgramConfigMissing) => {} + Err(err) => return Err(unit_error(err.into())), + } + } + import + } None => { // Synthesize the AudioSpecificConfig `description` so out-of-band consumers // (fMP4/MKV export, WebCodecs) can configure the decoder. A channel_config of 0 @@ -2326,16 +2350,23 @@ impl AacStream { header.channel_config, &mut block, ) - .map_err(|err| unit_error(err.into()))?; + .map_err(|err| { + // ffmpeg writes the element in its first frame only, so a receiver that + // joined later may never see one. Stop withholding the catalog for this + // track; it joins the catalog if an element arrives. + self.reserved.take(); + unit_error(err.into()) + })?; let mut config = aac::config(&asc)?; config.container = self.container.clone(); // Consume the reservation held since the PMT: this resolves the gated rendition, // and carries the catalog's declared media retention onto the track. - let reserved = self.reserved.take().expect("aac reservation already consumed"); + let reserved = self.reserved.take().unwrap_or_else(|| self.catalog.reserve()); let track = self .broadcast .unique_track(".aac", reserved.track_info(hang::catalog::PRIORITY.audio))?; let aac = aac::Import::new(track, reserved, config)?; + self.asc = asc; self.import.insert(aac) } }; @@ -4800,6 +4831,132 @@ pub(super) mod test { drop(import); } + /// A receiver that joins an AAC stream after its program config element (ffmpeg writes it in + /// the first frame only) cannot build that track, but must not withhold the catalog for the + /// rest of the program. The track joins once an element arrives, and a repeated element + /// leaves its frame, since the description carries the layout. + #[tokio::test(start_paused = true)] + async fn aac_joined_after_its_program_config_publishes_the_rest() { + const AAC_PID: u16 = 0x0060; + // ffmpeg's quad layout, two channel pair elements in a program config element. + let mut quad = vec![0x11, 0x80, 0x04, 0xC4, 0x04, 0x00, 0x21, 0x10, 0x0C]; + quad.extend_from_slice(b"Lavc63.1.101"); + let pce = crate::codec::aac::in_band(&quad).unwrap().program_config.unwrap(); + // A raw data block: the element, if given, then bytes that lead with a channel pair (ID 1). + let frame = |pce: &[u8]| { + let mut f = super::adts::write_header(2, 48_000, 0, pce.len() + 10) + .unwrap() + .to_vec(); + f.extend_from_slice(pce); + f.resize(f.len() + 10, 0x20); + f + }; + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut updates = catalog.consume().unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + + let mut mux = Mux { + out: synth_pmt(&[(StreamType::H264, VIDEO), (StreamType::AdtsAac, AAC_PID)], false), + ..Default::default() + }; + mux.gop(VIDEO, 90_000); + let cc = mux.cc(AAC_PID); + let bare = frame(&[]); + mux.out.extend(audio_pes_packet( + AAC_PID, + cc, + 90_000, + &[bare.as_slice(), &bare, &bare].concat(), + )); + import.decode(&mux.out).unwrap(); + assert_eq!(import.stats().streams[&AAC_PID].damaged, 1); + + let first = tokio::time::timeout(Duration::from_millis(10), updates.next()) + .await + .expect("the catalog waits on a track that cannot resolve") + .unwrap() + .unwrap(); + assert_eq!(first.video.renditions.len(), 1); + assert!(first.audio.renditions.is_empty()); + + // The element arrives, then frames without it, then a repeat. + let with = frame(&pce); + let cc = mux.cc(AAC_PID); + let late = audio_pes_packet(AAC_PID, cc, 180_000, &[with.as_slice(), &bare, &with, &bare].concat()); + import.decode(&late).unwrap(); + import.finish().unwrap(); + + let audio = loop { + let update = tokio::time::timeout(Duration::from_millis(10), updates.next()) + .await + .expect("the AAC track never joined") + .unwrap() + .unwrap(); + if let Some(audio) = update.audio.renditions.values().next() { + break audio.clone(); + } + }; + assert_eq!(audio.channel_count, 4); + assert_eq!(audio.description.as_deref(), Some(quad.as_slice())); + + let frames = read_audio_frames(&consumer, &catalog).await; + assert_eq!(frames.len(), 4, "only the frames from the element on"); + for frame in frames { + assert_eq!(frame.payload.as_ref(), [0x20; 10], "the element leaves every frame"); + } + } + + // A program config element that changes the layout mid-stream refuses its unit as + // damaged, and the track carries on with the frames around it. + #[tokio::test] + async fn aac_program_config_changed_mid_stream_refuses_the_unit() { + const AAC_PID: u16 = 0x0060; + // ffmpeg's quad layout: one front and one back channel pair. + let mut quad = vec![0x11, 0x80, 0x04, 0xC4, 0x04, 0x00, 0x21, 0x10, 0x0C]; + quad.extend_from_slice(b"Lavc63.1.101"); + // The same two pairs, with the second moved from the back to the side. + let mut side = quad.clone(); + side[4] = 0x40; + let element = |asc: &[u8]| crate::codec::aac::in_band(asc).unwrap().program_config.unwrap(); + let frame = |pce: &[u8]| { + let mut f = super::adts::write_header(2, 48_000, 0, pce.len() + 10) + .unwrap() + .to_vec(); + f.extend_from_slice(pce); + f.resize(f.len() + 10, 0x20); + f + }; + let (with, changed, bare) = (frame(&element(&quad)), frame(&element(&side)), frame(&[])); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut import = super::Import::new(broadcast, catalog.reserve()); + + let mut mux = Mux { + out: synth_pmt(&[(StreamType::AdtsAac, AAC_PID)], false), + ..Default::default() + }; + for (pts, unit) in [ + (90_000, [with.as_slice(), &bare]), + (180_000, [changed.as_slice(), &bare]), + ] { + let cc = mux.cc(AAC_PID); + mux.out.extend(audio_pes_packet(AAC_PID, cc, pts, &unit.concat())); + } + let cc = mux.cc(AAC_PID); + mux.out.extend(audio_pes_packet(AAC_PID, cc, 270_000, &bare)); + import.decode(&mux.out).unwrap(); + import.finish().unwrap(); + assert_eq!(import.stats().streams[&AAC_PID].damaged, 1); + + let frames = read_audio_frames(&consumer, &catalog).await; + assert_eq!(frames.len(), 3, "the changed unit is dropped, the rest publish"); + } + // Resync is bounded: a PID carrying something other than the codec its PMT declares is a // config error, not damage, so it must still fail rather than scan forever behind an // unresolved catalog reservation (which would withhold the catalog for every track).