From a6020349ab854fda49878bb8bead2f7ad4e1b436 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 28 Oct 2026 23:14:44 -0700 Subject: [PATCH 1/4] chore(quest): claim quest/m2/moqsrc-opus-surround.md Quest-Claim: 22f27718be50c599 From 6bf9df1b419f23bb96240a55f2e5e7a9c00f5549 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 21:22:31 -0800 Subject: [PATCH 2/4] wip: checkpoint unpushed work from the 2026-10-29 session The session that claimed this quest stopped before committing. This preserves its uncommitted changes as found on 2026-11-02; they are unfinished and unreviewed. Co-Authored-By: Claude Opus 5.5 --- quest/m2/README.md | 1 - quest/m2/moqsrc-opus-surround.md | 33 ------- rs/moq-gst/src/source/imp.rs | 149 ++++++++++++++++++++++++++++++- 3 files changed, 145 insertions(+), 38 deletions(-) delete mode 100644 quest/m2/moqsrc-opus-surround.md diff --git a/quest/m2/README.md b/quest/m2/README.md index 1751b2c37dd..ef3e0337058 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -64,7 +64,6 @@ lite-07; bitrate enforcement is not part of this release. - [AudioToolbox decode](/quest/m2/audio-decode-audiotoolbox.md) - macOS and iOS decode HE-AAC, multichannel AAC, and what else the framework offers - [AudioToolbox encode](/quest/m2/audio-encode-audiotoolbox.md) - macOS and iOS encode AAC-LC - [CMAF surround Opus](/quest/m2/cmaf-opus-surround.md) - a released mp4-atom carries the `dOps` channel mapping table, and fMP4 import and export carry it through the OpusHead -- [moqsrc surround Opus](/quest/m2/moqsrc-opus-surround.md) - moqsrc sets the Opus mapping caps, so opusdec plays 3 to 8 channels - [GPU CI](/quest/m2/gpu-ci.md) - NVIDIA tests gate the merge queue on a self-hosted GPU runner VM, and `just rs nvidia` runs them locally instead of skipping - [VA-API renderer tests](/quest/m2/vaapi-render-tests.md) - the ignored VA-API renderer import tests run in `just rs gpu` on AMD and Intel and fail instead of skipping - [Vendor-free CARLA bridge](/quest/m2/carla-bridge.md) - the CARLA demo's GPU bridge lets moq-video pick the encoder, and its stand-in test runs on AMD as well as NVIDIA diff --git a/quest/m2/moqsrc-opus-surround.md b/quest/m2/moqsrc-opus-surround.md deleted file mode 100644 index 197e1e7d127..00000000000 --- a/quest/m2/moqsrc-opus-surround.md +++ /dev/null @@ -1,33 +0,0 @@ -# [XS] moqsrc plays surround Opus, under test - -## Goal - -A test proves that a 3 to 8 channel Opus rendition published through -`moqsink` plays back through `moqsrc` and `opusdec` with every channel. - -## Plan - -Decided 2026-10-22 (maintainer, from the release audit): m2. Receive never -worked, so this is not a regression. -[moq-dev/moq#4991](https://github.com/moq-dev/moq/pull/4991) fixed publishing -and named this follow-up. - -The caps side landed in -[moq-dev/moq#6284](https://github.com/moq-dev/moq/pull/6284): `opus_caps` in -`rs/moq-gst/src/source/imp.rs` parses the catalog's OpusHead with -`opus::Mapping`, sets `channel-mapping-family`, `stream-count`, -`coupled-count`, and `channel-mapping`, and sends the OpusHead plus an empty -OpusTags as the `streamheader`, since `opusdec` reads the head (and its output -gain) only from a two-buffer `streamheader`. Its unit test, -`opus_caps_carry_the_stream_layout`, covers only mono and stereo. - -What remains is the end-to-end test: `opusenc` 5.1 through `moqsink`, a -relay, `moqsrc`, and `opusdec`, checking six decoded channels, plus each -channel count from 3 to 8 in the caps unit test, since every one maps -through the same family-1 table. - -Public API: none. Wire: none. - -## Related - -- [CMAF surround Opus](/quest/m2/cmaf-opus-surround.md) - the same mapping table through fMP4 `dOps`, independent of this caps path diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 9b5c60f64e3..0304742bc40 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1924,6 +1924,61 @@ mod tests { } } + /// Every surround layout reaches opusdec as its family 1 mapping, RFC 7845's Vorbis order. + #[test] + fn surround_opus_caps_carry_the_family_1_mapping() { + gst::init().unwrap(); + let layouts: [(u8, u8, &[u8]); 6] = [ + (2, 1, &[0, 2, 1]), + (2, 2, &[0, 1, 2, 3]), + (3, 2, &[0, 4, 1, 2, 3]), + (4, 2, &[0, 4, 1, 2, 3, 5]), + (4, 3, &[0, 4, 1, 2, 3, 5, 6]), + (5, 3, &[0, 6, 1, 2, 3, 4, 5, 7]), + ]; + for (streams, coupled, table) in layouts { + let channels = table.len() as u32; + let mut head = moq_mux::codec::opus::Config::new(48_000, channels).with_pre_skip(312); + head.mapping = Some( + moq_mux::codec::opus::Mapping::new(moq_mux::codec::opus::mapping::Config { + family: 1, + streams, + coupled, + table, + }) + .unwrap(), + ); + let description = head.encode().unwrap(); + let mut config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, channels); + config.description = Some(description.clone()); + let caps = super::audio_caps(&config).unwrap(); + let structure = caps.structure(0).unwrap(); + assert_eq!(structure.get::("channels").unwrap(), channels as i32); + assert_eq!(structure.get::("channel-mapping-family").unwrap(), 1); + assert_eq!(structure.get::("stream-count").unwrap(), i32::from(streams)); + assert_eq!(structure.get::("coupled-count").unwrap(), i32::from(coupled)); + let mapping: Vec = structure + .get::("channel-mapping") + .unwrap() + .iter() + .map(|entry| entry.get::().unwrap()) + .collect(); + assert_eq!( + mapping, + table.iter().map(|&c| i32::from(c)).collect::>(), + "{channels} channels" + ); + let head: gst::Buffer = structure + .get::("streamheader") + .unwrap() + .first() + .unwrap() + .get() + .unwrap(); + assert_eq!(head.map_readable().unwrap().as_slice(), description.as_ref()); + } + } + /// opusdec only clips what a buffer's meta asks, one packet at a time, and stamps what's left /// at the packet's pts, so the pre-skip spreads over as many packets as it spans and every /// packet moves to where its first kept sample plays. @@ -2021,7 +2076,10 @@ mod session_tests { use hang::catalog::{AudioCodec, AudioConfig, Container, H264, VideoCodec, VideoConfig}; use tokio::sync::watch; - use super::{NEXT_VIDEO_PAD_ID, Pads, ResolvedSettings, SessionController, TrackKind, follow_catalog, follow_path}; + use super::{ + NEXT_AUDIO_PAD_ID, NEXT_VIDEO_PAD_ID, Pads, ResolvedSettings, SessionController, TrackKind, follow_catalog, + follow_path, + }; /// The pad-id counters are process-global, so a test reading one has to be the only test /// allocating while it runs. `cargo test` shares a process across tests (nextest doesn't), @@ -4584,15 +4642,20 @@ mod session_tests { } } - /// `moqsrc` playing `room` from `url`, its first video pad linked by name to a synced sink - /// that drops anything more than 100ms late. - fn pipeline(url: &url::Url) -> (gst::Pipeline, gst::Element) { + /// Makes both elements available to pipeline descriptions. + fn register() { static REGISTER: std::sync::Once = std::sync::Once::new(); gst::init().unwrap(); REGISTER.call_once(|| { gst::Element::register(None, "moqsrc", gst::Rank::NONE, super::super::MoqSrc::static_type()).unwrap(); + gst::Element::register(None, "moqsink", gst::Rank::NONE, crate::sink::MoqSink::static_type()).unwrap(); }); + } + /// `moqsrc` playing `room` from `url`, its first video pad linked by name to a synced sink + /// that drops anything more than 100ms late. + fn pipeline(url: &url::Url) -> (gst::Pipeline, gst::Element) { + register(); let first = NEXT_VIDEO_PAD_ID.load(Ordering::Relaxed); let pipeline = gst::parse::launch(&format!( "moqsrc name=src url={url} broadcast=room \ @@ -4940,4 +5003,82 @@ mod session_tests { rendered(&sink, &mut new, 2); assert_played_cleanly(pipeline); } + + /// The surround path end to end: `opusenc` 5.1 through moqsink, a relay, moqsrc, and + /// `opusdec` decodes six channels, each carrying the tone. The tone sits low enough for the + /// LFE channel, which libopus band-limits, to carry it too. + #[test] + fn surround_opus_plays_every_channel_through_a_relay() { + const CHANNELS: usize = 6; + // Half a second at 48 kHz. + const FRAMES: usize = 24_000; + + let _pad_ids = pad_ids(); + let relay = Relay::new(); + register(); + let launch = |description: String| { + gst::parse::launch(&description) + .unwrap() + .downcast::() + .unwrap() + }; + let publisher = launch(format!( + "audiotestsrc is-live=true freq=100 \ + ! audio/x-raw,format=S16LE,layout=interleaved,rate=48000,channels={CHANNELS},channel-mask=(bitmask)0x3f \ + ! opusenc ! moqsink url={} broadcast=room", + relay.url + )); + let first = NEXT_AUDIO_PAD_ID.load(Ordering::Relaxed); + let subscriber = launch(format!( + "moqsrc name=src url={} broadcast=room src.audio_{first} ! opusdec ! appsink name=sink sync=false", + relay.url + )); + let sink = subscriber.by_name("sink").unwrap(); + publisher.set_state(gst::State::Playing).unwrap(); + subscriber.set_state(gst::State::Playing).unwrap(); + + let errors = || { + [&publisher, &subscriber].map(|pipeline| pipeline.bus().unwrap().pop_filtered(&[gst::MessageType::Error])) + }; + let deadline = std::time::Instant::now() + Duration::from_secs(30); + let mut samples = Vec::new(); + while samples.len() < FRAMES * CHANNELS { + assert!( + std::time::Instant::now() < deadline, + "{} of {FRAMES} frames decoded; errors: {:?}", + samples.len() / CHANNELS, + errors() + ); + let Some(sample) = sink.emit_by_name::>("try-pull-sample", &[&100_000_000u64]) else { + continue; + }; + let info = gst_audio::AudioInfo::from_caps(sample.caps().unwrap()).unwrap(); + assert_eq!(info.channels() as usize, CHANNELS, "opusdec decoded {info:?}"); + assert_eq!(info.format(), gst_audio::AudioFormat::S16le); + let buffer = sample.buffer().unwrap().map_readable().unwrap(); + samples.extend( + buffer + .chunks_exact(2) + .map(|sample| f64::from(i16::from_le_bytes([sample[0], sample[1]]))), + ); + } + subscriber.set_state(gst::State::Null).unwrap(); + publisher.set_state(gst::State::Null).unwrap(); + relay.stop(); + let errors = errors(); + assert!(errors.iter().all(Option::is_none), "{errors:?}"); + + // audiotestsrc writes the same tone to every channel at 0.8 of full scale, an RMS near + // 18,500; a channel the mapping dropped decodes to silence. + let frames = (samples.len() / CHANNELS) as f64; + for channel in 0..CHANNELS { + let energy = samples + .iter() + .skip(channel) + .step_by(CHANNELS) + .map(|sample| sample * sample); + let rms = (energy.sum::() / frames).sqrt(); + assert!(rms > 5_000.0, "channel {channel} decoded an RMS of {rms:.0}"); + } + } } From fff82f618deee4360add6f625ceb6e490dd8f242 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 09:57:43 -0800 Subject: [PATCH 3/4] test(moq-gst): give each surround channel its own tone A tone shared by every channel can't tell a dropped, duplicated, or swapped channel from a correct mapping. Feed six distinct tones through appsrc and check each decoded channel carries only its own. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 86 ++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 19 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 45bb8822876..7f121a60e34 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -5462,13 +5462,22 @@ mod session_tests { } /// The surround path end to end: `opusenc` 5.1 through moqsink, a relay, moqsrc, and - /// `opusdec` decodes six channels, each carrying the tone. The tone sits low enough for the - /// LFE channel, which libopus band-limits, to carry it too. + /// `opusdec` decodes six channels, each carrying its own tone, so a mapping that drops, + /// duplicates, or swaps a channel fails. The LFE tone sits low enough to survive the band + /// limit libopus applies to that channel. #[test] fn surround_opus_plays_every_channel_through_a_relay() { - const CHANNELS: usize = 6; - // Half a second at 48 kHz. - const FRAMES: usize = 24_000; + const RATE: usize = 48_000; + // GStreamer's order for channel-mask 0x3f: FL, FR, FC, LFE, RL, RR. Every tone completes + // whole cycles in the analysis window. + const TONES: [f64; 6] = [440.0, 660.0, 880.0, 60.0, 1_100.0, 1_320.0]; + const CHANNELS: usize = TONES.len(); + const AMPLITUDE: f64 = 8_000.0; + // Analyze the last half second of the first second decoded, past the encoder's warm-up. + const FRAMES: usize = RATE; + const WINDOW: usize = RATE / 2; + // 10 ms per pushed buffer. + const CHUNK: usize = RATE / 100; let _pad_ids = pad_ids(); let relay = Relay::new(); @@ -5479,10 +5488,11 @@ mod session_tests { .downcast::() .unwrap() }; + // The identity paces the source to the clock, like a live capture. let publisher = launch(format!( - "audiotestsrc is-live=true freq=100 \ - ! audio/x-raw,format=S16LE,layout=interleaved,rate=48000,channels={CHANNELS},channel-mask=(bitmask)0x3f \ - ! opusenc ! moqsink url={} broadcast=room", + "appsrc name=raw is-live=true format=time block=true \ + caps=audio/x-raw,format=S16LE,layout=interleaved,rate={RATE},channels={CHANNELS},channel-mask=(bitmask)0x3f \ + ! identity sync=true ! opusenc bitrate=256000 ! moqsink url={} broadcast=room", relay.url )); let first = NEXT_AUDIO_PAD_ID.load(Ordering::Relaxed); @@ -5494,6 +5504,31 @@ mod session_tests { publisher.set_state(gst::State::Playing).unwrap(); subscriber.set_state(gst::State::Playing).unwrap(); + // Push tones until the pipeline stops taking buffers. + let raw = publisher.by_name("raw").unwrap(); + let feeder = std::thread::spawn(move || { + for chunk in 0u64.. { + let start = chunk as usize * CHUNK; + let mut data = Vec::with_capacity(CHUNK * CHANNELS * 2); + for frame in start..start + CHUNK { + let time = frame as f64 / RATE as f64; + for tone in TONES { + let sample = (AMPLITUDE * (std::f64::consts::TAU * tone * time).sin()) as i16; + data.extend_from_slice(&sample.to_le_bytes()); + } + } + let mut buffer = gst::Buffer::from_mut_slice(data); + { + let buffer = buffer.get_mut().unwrap(); + buffer.set_pts(gst::ClockTime::SECOND.mul_div_floor(start as u64, RATE as u64)); + buffer.set_duration(gst::ClockTime::SECOND.mul_div_floor(CHUNK as u64, RATE as u64)); + } + if raw.emit_by_name::("push-buffer", &[&buffer]) != gst::FlowReturn::Ok { + break; + } + } + }); + let errors = || { [&publisher, &subscriber].map(|pipeline| pipeline.bus().unwrap().pop_filtered(&[gst::MessageType::Error])) }; @@ -5521,21 +5556,34 @@ mod session_tests { } subscriber.set_state(gst::State::Null).unwrap(); publisher.set_state(gst::State::Null).unwrap(); + feeder.join().unwrap(); relay.stop(); let errors = errors(); assert!(errors.iter().all(Option::is_none), "{errors:?}"); - // audiotestsrc writes the same tone to every channel at 0.8 of full scale, an RMS near - // 18,500; a channel the mapping dropped decodes to silence. - let frames = (samples.len() / CHANNELS) as f64; - for channel in 0..CHANNELS { - let energy = samples - .iter() - .skip(channel) - .step_by(CHANNELS) - .map(|sample| sample * sample); - let rms = (energy.sum::() / frames).sqrt(); - assert!(rms > 5_000.0, "channel {channel} decoded an RMS of {rms:.0}"); + // Each tone's amplitude in a channel, by the Goertzel algorithm. + let window = &samples[(FRAMES - WINDOW) * CHANNELS..FRAMES * CHANNELS]; + let amplitude = |channel: usize, tone: f64| { + let coeff = 2.0 * (std::f64::consts::TAU * tone / RATE as f64).cos(); + let (mut prev, mut prev2) = (0.0, 0.0); + for &sample in window.iter().skip(channel).step_by(CHANNELS) { + (prev, prev2) = (sample + coeff * prev - prev2, prev); + } + 2.0 * (prev * prev + prev2 * prev2 - coeff * prev * prev2).sqrt() / WINDOW as f64 + }; + for (channel, own) in TONES.into_iter().enumerate() { + let heard = TONES.map(|tone| amplitude(channel, tone).round()); + assert!( + heard[channel] > AMPLITUDE / 2.0, + "channel {channel} lost its {own} Hz tone: {heard:?}" + ); + for (other, &level) in heard.iter().enumerate() { + assert!( + other == channel || level < AMPLITUDE / 20.0, + "channel {channel} carries channel {other}'s {} Hz tone: {heard:?}", + TONES[other] + ); + } } } } From 3122c4aa9dd3f8ee60ef1fc1176022dbd89921c7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 10:01:27 -0800 Subject: [PATCH 4/4] test(moq-gst): read the deadline from kio's clock Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 7f121a60e34..8902df6a38a 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -5532,11 +5532,11 @@ mod session_tests { let errors = || { [&publisher, &subscriber].map(|pipeline| pipeline.bus().unwrap().pop_filtered(&[gst::MessageType::Error])) }; - let deadline = std::time::Instant::now() + Duration::from_secs(30); + let deadline = kio::clock::now() + Duration::from_secs(30); let mut samples = Vec::new(); while samples.len() < FRAMES * CHANNELS { assert!( - std::time::Instant::now() < deadline, + kio::clock::now() < deadline, "{} of {FRAMES} frames decoded; errors: {:?}", samples.len() / CHANNELS, errors() @@ -5550,8 +5550,10 @@ mod session_tests { let buffer = sample.buffer().unwrap().map_readable().unwrap(); samples.extend( buffer - .chunks_exact(2) - .map(|sample| f64::from(i16::from_le_bytes([sample[0], sample[1]]))), + .as_chunks::<2>() + .0 + .iter() + .map(|&sample| f64::from(i16::from_le_bytes(sample))), ); } subscriber.set_state(gst::State::Null).unwrap();