diff --git a/doc/bin/cli.md b/doc/bin/cli.md index edd7ae84772..fa3663dd3dc 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -138,6 +138,10 @@ MPEG-TS export frames AAC as ADTS, which labels only the AAC Main, LC, SSR, and LTP profiles. HE-AAC and HE-AACv2 go out as their AAC-LC core, and decoders find the SBR and PS in band, as ffmpeg's ADTS output does. A track whose profile or channel layout ADTS cannot label is refused rather than mislabeled. +An Opus track is labeled with the plain channel code its extension descriptor +can name: a family 0 head, a family 1 head with the Vorbis mapping, or mono or +stereo when the track has no OpusHead. Any other head is refused rather than +written with a guessed channel code. A constant-rate MPEG-TS source records its multiplex rate in the catalog (`mpegts.muxRate`, measured off the PCR clock, null stuffing included), and diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index 8e9b7a838df..125989f0866 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -14,7 +14,7 @@ Turns existing container formats into hang broadcasts and back. This is what | Format | Import | Export | Notes | | --- | --- | --- | --- | | fMP4 / CMAF | yes | yes | Passthrough as `cmaf` or repackaged as `legacy`. | -| MPEG-TS | yes | yes | H.264/H.265; AAC, MP2, AC-3, E-AC-3, Opus up to 7.1; SCTE-35 and subtitle PIDs carried as tracks; service tables round-trip; signalled timebase discontinuities preserved; paced export. | +| MPEG-TS | yes | yes | H.264/H.265; AAC, MP2, AC-3, E-AC-3, Opus up to 7.1 for a Vorbis head (other mappings refused); SCTE-35 and subtitle PIDs carried as tracks; service tables round-trip; signalled timebase discontinuities preserved; paced export. | | FLV / RTMP | yes | yes | Legacy H.264 + AAC + MP3, plus enhanced-RTMP HEVC, AV1, VP9, Opus, AC-3, E-AC-3, and multitrack. | | Matroska / WebM | yes | yes | | | Annex-B (H.264, H.265) | yes | yes | Parameter sets extracted to the catalog or re-injected per keyframe. | diff --git a/quest/m1/README.md b/quest/m1/README.md index b4ac1f6d833..7ae4ab69c27 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -102,7 +102,6 @@ blocks. The quests that gated m0 lines moved under them. - [OBS native codecs](/quest/m1/obs-moq-video/README.md) - replace FFmpeg video and audio decoding with moq-video and moq-audio, deliver GPU frames, and use native audio/video encoders - [AudioToolbox decode](/quest/m1/audio-decode-audiotoolbox.md) - macOS and iOS decode HE-AAC, multichannel AAC, and what else the framework offers - [HE-AAC catalog output](/quest/m1/he-aac-catalog-output.md) - HE-AAC catalog entries name the output rate and layout, not the LC core -- [TS Opus export refusals](/quest/m1/ts-opus-export-refusals.md) - the TS exporter refuses Opus heads its channel code cannot describe instead of mislabeling them - [TS Opus channel codes](/quest/m1/ts-opus-channel-codes.md) - TS Opus with a channel code of 0x81 or above imports with its real head or is refused, never guessed as stereo - [AudioToolbox encode](/quest/m1/audio-encode-audiotoolbox.md) - macOS and iOS encode AAC-LC - [AAC encode refusals](/quest/m1/aac-encode-refusals.md) - `Config::encode` refuses channel counts it cannot name instead of writing stereo diff --git a/quest/m1/ts-opus-channel-codes.md b/quest/m1/ts-opus-channel-codes.md index 221cc675592..4bb13e9e320 100644 --- a/quest/m1/ts-opus-channel-codes.md +++ b/quest/m1/ts-opus-channel-codes.md @@ -21,7 +21,3 @@ multistream packet. otherwise, noted as such. Public API: none. Wire: none. - -## Related - -- [TS Opus export refusals](/quest/m1/ts-opus-export-refusals.md) - the exporter side of the same descriptor diff --git a/quest/m1/ts-opus-export-refusals.md b/quest/m1/ts-opus-export-refusals.md deleted file mode 100644 index b376124fbf3..00000000000 --- a/quest/m1/ts-opus-export-refusals.md +++ /dev/null @@ -1,23 +0,0 @@ -# [S] TS Opus export refuses what it cannot label - -## Goal - -`moq-mux`'s MPEG-TS exporter never labels an Opus track with a channel -configuration its packets do not have. A track whose OpusHead the Opus -extension descriptor can describe exports as today; anything else is refused, -not written with a guessed `channel_config_code`. - -## Plan - -- The exporter derives `channel_config_code` from the catalog channel count - alone, clamped to 1..=8, so a family 255 or ambisonics head, a family 1 head - with a non-Vorbis table, or more than eight channels exports mislabeled. - Decide from the parsed description: family 0, and family 1 with the Vorbis - default mapping, keep the plain code; refuse the rest, or write the explicit - layout if the import side of - [TS Opus channel codes](/quest/m1/ts-opus-channel-codes.md) settles - one both sides agree on. -- A track with no description stays mono or stereo only. -- Tests with real heads, checked against ffprobe where ffmpeg reads the result. - -Public API: none. Wire: none; TS output changes only for inputs it mislabeled. diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 50fa3979d38..d679ad27d14 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -38,7 +38,7 @@ use moq_net::Timestamp; use crate::catalog::hang::Catalog; use crate::catalog::{CatalogFormat, Stream}; use crate::codec::video::Reorder; -use crate::codec::{aac, annexb}; +use crate::codec::{aac, annexb, opus}; use crate::container::{ExportSource, Frame}; use super::adts; @@ -407,8 +407,9 @@ enum Kind { Aac(aac::InBand), /// Opus (private stream_type 0x06). Each frame is one Opus packet, prefixed with /// the Opus-in-TS access-unit control header and announced with the 'Opus' - /// registration plus DVB extension descriptor. - Opus { channel_count: u32 }, + /// registration plus DVB extension descriptor. `channel_config_code` is a plain + /// code that descriptor can name, never a count clamped into range. + Opus { channel_config_code: u8 }, /// MP2, carried verbatim. The sample rate picks the stream type on the way /// out (0x03 vs 0x04). Mp2 { sample_rate: u32 }, @@ -1528,7 +1529,7 @@ impl Export { tag: 0x05, data: b"EAC3".to_vec(), }], - Kind::Opus { channel_count } => opus_descriptors(*channel_count), + Kind::Opus { channel_config_code } => opus_descriptors(*channel_config_code), _ => Vec::new(), } }; @@ -2449,7 +2450,7 @@ fn audio_kind(config: &AudioConfig, name: &str) -> anyhow::Result { sample_rate: config.sample_rate, }), AudioCodec::Opus => Ok(Kind::Opus { - channel_count: config.channel_count, + channel_config_code: opus_channel_code(config, name)?, }), AudioCodec::Ac3 => Ok(Kind::Ac3), AudioCodec::Ec3 => Ok(Kind::Eac3), @@ -2460,7 +2461,10 @@ fn audio_kind(config: &AudioConfig, name: &str) -> anyhow::Result { /// The two PMT descriptors for an Opus elementary stream: the `Opus` registration /// descriptor (which sets the codec) and the DVB extension descriptor 0x80 carrying /// the channel configuration. ffmpeg's demuxer requires both to recognize the stream. -fn opus_descriptors(channel_count: u32) -> Vec { +/// +/// `channel_config_code` is already a plain code ([`opus_channel_code`]): 1 is mono, +/// 2 is stereo, and 3..=8 is the Vorbis family 1 mapping for that many channels. +fn opus_descriptors(channel_config_code: u8) -> Vec { vec![ Descriptor { tag: 0x05, @@ -2468,13 +2472,60 @@ fn opus_descriptors(channel_count: u32) -> Vec { }, Descriptor { tag: 0x7f, - // extension_descriptor_tag 0x80, then channel_config_code (1=mono, 2=stereo, - // = channel count for the Vorbis mapping), clamped to the 1..=8 the demuxer reads. - data: vec![0x80, channel_count.clamp(1, 8) as u8], + // extension_descriptor_tag 0x80, then the plain channel_config_code. + data: vec![0x80, channel_config_code], }, ] } +/// Plain `channel_config_code` for an Opus track the extension descriptor can name. +/// +/// Family 0, and family 1 with the Vorbis mapping, use that channel count. A missing +/// head is mono or stereo only. Every other layout is refused: the descriptor has no +/// code for it, and a clamped count would name a layout the packets do not have. +fn opus_channel_code(config: &AudioConfig, name: &str) -> anyhow::Result { + let Some(description) = config.description.as_deref() else { + anyhow::ensure!( + matches!(config.channel_count, 1 | 2), + "TS export cannot label Opus track '{name}' with {} channels and no OpusHead", + config.channel_count + ); + return Ok(config.channel_count as u8); + }; + + let mut buf = description; + let head = opus::Config::parse(&mut buf) + .map_err(|err| anyhow::anyhow!("TS export cannot read the OpusHead on track '{name}': {err}"))?; + anyhow::ensure!( + head.channel_count == config.channel_count, + "Opus head has {} channels but the catalog declares {} (track '{name}')", + head.channel_count, + config.channel_count + ); + + if let Some(mapping) = &head.mapping { + let channels = mapping.table().len() as u8; + let vorbis = opus::Mapping::vorbis(channels).ok(); + anyhow::ensure!( + vorbis.as_ref() == Some(mapping), + "TS export cannot label Opus track '{name}': channel mapping family {} is not the Vorbis layout", + mapping.family() + ); + } + + let code = u8::try_from(head.channel_count).with_context(|| { + format!( + "TS export cannot label Opus track '{name}' with {} channels", + head.channel_count + ) + })?; + anyhow::ensure!( + (1..=8).contains(&code), + "TS export cannot label Opus track '{name}' with {code} channels" + ); + Ok(code) +} + /// Wrap a raw Opus packet in the Opus-in-TS access-unit control header, producing one /// PES access unit. Emits the 11-bit `0x3FF` sync (no trim, no control extension), then /// the `0xFF`-run `au_size`, then the packet. diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 5f1dd5bfe65..31e540f3a1e 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -6,7 +6,7 @@ //! parameter sets on keyframes). These build a synthetic broadcast, export to //! TS, and re-parse with the `mpeg2ts` reader. -use std::io::Cursor; +use std::io::{Cursor, Write}; use std::time::Duration; use bytes::{Bytes, BytesMut}; @@ -3786,6 +3786,182 @@ async fn opus_export_import_roundtrip() { } } +/// An OpusHead for `channels` channels. `pre_skip` is libopus's usual 6.5 ms, so the +/// bytes are a real head rather than a channel count with the magic glued on. +fn opus_head(channels: u32, mapping: Option) -> Bytes { + let mut config = crate::codec::opus::Config::new(48_000, channels); + config.pre_skip = 312; + config.mapping = mapping; + config.encode().expect("a real OpusHead") +} + +fn opus_mapping(family: u8, streams: u8, coupled: u8, table: &[u8]) -> crate::codec::opus::Mapping { + crate::codec::opus::Mapping::new(crate::codec::opus::mapping::Config { + family, + streams, + coupled, + table, + }) + .expect("mapping") +} + +/// Publish one Opus frame and export it. `Err` is the exporter's refusal. +async fn export_opus_track(channel_count: u32, description: Option) -> Result { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + + let track = broadcast + .create_track( + broadcast.unique_name(".opus"), + hang::container::track_info(hang::catalog::PRIORITY.audio), + ) + .unwrap(); + let mut cfg = AudioConfig::new(AudioCodec::Opus, 48_000, channel_count); + cfg.container = Container::Legacy; + cfg.description = description; + catalog + .modify() + .unwrap() + .audio + .renditions + .insert(track.name().to_string(), cfg); + + let mut producer = Producer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); + producer + .write(Frame { + timestamp: Timestamp::ZERO, + duration: None, + payload: opus_packet(0x01, 8), + keyframe: true, + }) + .unwrap(); + producer.finish().unwrap(); + + let mut exporter = Export::new(crate::source::announced(&consumer)) + .await + .unwrap() + .with_max_age(RECORDING_MAX_AGE); + let mut out = BytesMut::new(); + loop { + match tokio::time::timeout(Duration::from_secs(1), exporter.next()).await { + Ok(Ok(Some(frame))) => out.extend_from_slice(&frame.payload), + Ok(Ok(None)) => return Ok(out), + Ok(Err(err)) => return Err(err.to_string()), + Err(_) => return Ok(out), + } + } +} + +/// The plain `channel_config_code` on the first PMT. +fn opus_config_code(ts: &[u8]) -> u8 { + let mut reader = TsPacketReader::new(Cursor::new(ts)); + while let Some(packet) = reader.read_ts_packet().unwrap() { + if let Some(TsPayload::Pmt(pmt)) = packet.payload { + let ext = pmt.es_info[0] + .descriptors + .iter() + .find(|d| d.tag == 0x7f) + .expect("extension descriptor"); + assert_eq!(ext.data[0], 0x80, "extension_descriptor_tag"); + return ext.data[1]; + } + } + panic!("missing PMT"); +} + +/// Channel count ffprobe reads from the descriptor. ffmpeg's demuxer is what a plain +/// code has to agree with; a clamped or guessed code shows up here as the wrong count. +fn ffprobe_opus_channels(ts: &[u8]) -> u32 { + let mut child = std::process::Command::new("ffprobe") + .args([ + "-v", + "error", + "-select_streams", + "a:0", + "-show_entries", + "stream=codec_name,channels", + "-of", + "csv=p=0", + "-i", + "pipe:0", + ]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .expect("ffprobe is in the dev shell"); + child.stdin.take().unwrap().write_all(ts).unwrap(); + let output = child.wait_with_output().unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "ffprobe failed: {stderr} stdout={stdout}"); + let line = stdout.lines().next().unwrap_or("").trim(); + let (codec, channels) = line.split_once(',').unwrap_or_else(|| panic!("ffprobe: {line}")); + assert_eq!(codec, "opus", "ffprobe: {line} ({stderr})"); + channels + .trim() + .parse() + .unwrap_or_else(|_| panic!("ffprobe channels in {line}")) +} + +/// A head the extension descriptor can name keeps the plain channel code, and +/// ffprobe reads that count. A family 255 or ambisonic head, a family 1 table that +/// is not Vorbis, a head that contradicts the catalog, or more than stereo with no +/// head is refused rather than labeled with a clamped count. +#[tokio::test(start_paused = true)] +async fn opus_export_refuses_a_head_it_cannot_label() { + for channels in 1..=2 { + let ts = export_opus_track(channels, Some(opus_head(channels, None))) + .await + .expect("family 0"); + assert_eq!(opus_config_code(&ts), channels as u8); + assert_eq!(ffprobe_opus_channels(&ts), channels); + let ts = export_opus_track(channels, None).await.expect("no head"); + assert_eq!(opus_config_code(&ts), channels as u8); + assert_eq!(ffprobe_opus_channels(&ts), channels); + } + for channels in 1..=8u32 { + let mapping = crate::codec::opus::Mapping::vorbis(channels as u8).unwrap(); + let ts = export_opus_track(channels, Some(opus_head(channels, Some(mapping)))) + .await + .unwrap_or_else(|err| panic!("{channels} channel Vorbis head: {err}")); + assert_eq!(opus_config_code(&ts), channels as u8, "{channels} channels"); + assert_eq!(ffprobe_opus_channels(&ts), channels, "{channels} channels"); + } + + // 5.1 with the center and LFE swapped, still four streams and two coupled. + let swapped = opus_mapping(1, 4, 2, &[0, 4, 1, 2, 5, 3]); + // The uncoupled identity table ffmpeg writes as `0x80 | channels`, which its demuxer does not read. + let identity = opus_mapping(1, 6, 0, &[0, 1, 2, 3, 4, 5]); + let ambisonic = opus_mapping(2, 4, 0, &[0, 1, 2, 3]); + let family_255 = opus_mapping(255, 2, 0, &[0, 1]); + let wide = opus_mapping(255, 9, 0, &[0, 1, 2, 3, 4, 5, 6, 7, 8]); + + let refused: [(u32, Option, &str); 9] = [ + (0, None, "no OpusHead"), + (6, None, "no OpusHead"), + (9, None, "no OpusHead"), + (6, Some(opus_head(6, Some(swapped))), "not the Vorbis layout"), + (6, Some(opus_head(6, Some(identity))), "not the Vorbis layout"), + (4, Some(opus_head(4, Some(ambisonic))), "not the Vorbis layout"), + (2, Some(opus_head(2, Some(family_255))), "not the Vorbis layout"), + (9, Some(opus_head(9, Some(wide))), "not the Vorbis layout"), + (6, Some(opus_head(2, None)), "catalog declares"), + ]; + for (channels, description, needle) in refused { + let err = export_opus_track(channels, description) + .await + .expect_err("a guessed channel_config_code"); + assert!(err.contains(needle), "expected {needle} in {err}"); + } + + let err = export_opus_track(2, Some(Bytes::from_static(b"not-an-opus-head"))) + .await + .expect_err("a head that does not parse"); + assert!(err.contains("cannot read the OpusHead"), "{err}"); +} + // Two exporters of one broadcast, started at different times, must render the same packets // from the moment they overlap. That is what a redundant (SMPTE ST 2022-7) pair compares, and // it is what lets a leg be restarted without the merge at the far end seeing the two disagree.