From c47a44496ec5aefacf29ee0e7f3c9df8d4460e0f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 21:24:45 -0700 Subject: [PATCH 1/8] chore(quest): claim quest/m1/interop-preskip.md Quest-Claim: fc777492260dbd1c From f50317f363985b4bd126db10fb9f42bdaec1e010 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 22 Oct 2026 23:26:37 -0700 Subject: [PATCH 2/8] test(interop): hold the tone's phase to its timestamps, and clip Opus pre-skip in moqsrc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tone.py --phase requires the sample decoded at timestamp t to carry phase 2πft, so a decoder that keeps or drops the wrong number of priming samples fails. It runs for every pair but the browser publisher and the native-JS subscriber. It caught moqsrc: opusdec trims pre-skip only from GstAudioClippingMeta, which moqsrc never attached, so GStreamer played 312 priming samples and its audio ran 6.5 ms late. moqsrc now clips the OpusHead's pre-skip across the packets it spans and stamps each packet where its first kept sample plays. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 77 ++++++++++++++++++++++++--- quest/m1/README.md | 1 - quest/m1/interop-aac.md | 4 +- quest/m1/interop-preskip.md | 34 ------------ rs/moq-gst/Cargo.toml | 1 + rs/moq-gst/src/source/imp.rs | 90 +++++++++++++++++++++++++++++++- rs/moq-mux/src/codec/opus/mod.rs | 9 ++-- test/interop/README.md | 8 ++- test/interop/interop.sh | 15 +++++- test/interop/tone.py | 51 ++++++++++++++---- test/interop/tone.test.py | 27 +++++++--- 11 files changed, 248 insertions(+), 69 deletions(-) delete mode 100644 quest/m1/interop-preskip.md diff --git a/Cargo.lock b/Cargo.lock index b8cb020c2fd..c67cb2fea6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -581,6 +581,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "atomic_refcell" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21e4227379beff4205943696e6c3e0cd809bacdf3f0edd6e3dd153e2269571a4" + [[package]] name = "attohttpc" version = "0.30.1" @@ -1959,7 +1965,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c54e03a951783e8b327515db3f2a2fd0e3bed362a96b066f341ce66ed49b4ead" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 3.0.6", ] [[package]] @@ -2228,7 +2234,7 @@ checksum = "263c0ccfc6b3b6a93e959edad4f985779e76c0b36dbb2e13e3d27c690974adf1" dependencies = [ "cfg-if", "libc", - "socket2 0.5.10", + "socket2 0.6.5", "windows-sys 0.61.2", ] @@ -3051,6 +3057,62 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "gstreamer-audio" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "544f14e19e3b7221845be0e5c2f2cf284576b7baa16835ac16def6e734bfa866" +dependencies = [ + "cfg-if", + "glib", + "gstreamer", + "gstreamer-audio-sys", + "gstreamer-base", + "libc", + "smallvec", +] + +[[package]] +name = "gstreamer-audio-sys" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a010deb715bc29953a07c6714249132e736b3daf3a529e70586b58a7063888" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-base-sys", + "gstreamer-sys", + "libc", + "system-deps 9.0.0", +] + +[[package]] +name = "gstreamer-base" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cbd8f59586a369c40d4b0d2f3a61cf9e9287b3cf5503cdcd8e7c5faa99265e" +dependencies = [ + "atomic_refcell", + "cfg-if", + "glib", + "gstreamer", + "gstreamer-base-sys", + "libc", +] + +[[package]] +name = "gstreamer-base-sys" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b05f77bfd76b5afee6b29c27dae19eaf5018b5cf8d0d17520f3de44c6385c32" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-sys", + "libc", + "system-deps 9.0.0", +] + [[package]] name = "gstreamer-sys" version = "0.25.4" @@ -4844,6 +4906,7 @@ dependencies = [ "futures", "gst-plugin-version-helper", "gstreamer", + "gstreamer-audio", "hang", "moq-audio", "moq-clock", @@ -5959,7 +6022,7 @@ dependencies = [ "pin-project-lite", "rustc-hash 2.1.3", "rustls", - "socket2 0.5.10", + "socket2 0.6.5", "thiserror 2.0.21", "tokio", "tokio-stream", @@ -6002,7 +6065,7 @@ checksum = "4dc50afea61aff926e150a36c31f06094608848e114a3fe667d76ec233163b1c" dependencies = [ "cfg_aliases", "libc", - "socket2 0.5.10", + "socket2 0.6.5", "tracing", "windows-sys 0.61.2", ] @@ -7612,7 +7675,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.3", "rustls", - "socket2 0.5.10", + "socket2 0.6.5", "thiserror 2.0.21", "tokio", "tracing", @@ -7651,7 +7714,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.5", "tracing", "windows-sys 0.61.2", ] @@ -9661,7 +9724,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix 1.1.5", "windows-sys 0.61.2", diff --git a/quest/m1/README.md b/quest/m1/README.md index afb1c5d6779..717b183004a 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -143,7 +143,6 @@ tracks, so per-group stats stop costing cluster announces. - [libav polyfill fork fix](/quest/m1/libav-polyfill-fork.md) - our polyfill fork copies planar audio correctly and `@moq/hang` drops its patch - [Wire-compat audio](/quest/m1/wire-compat-audio.md) - the wire-compat media lanes decode and check the interop tone across releases - [Interop AAC](/quest/m1/interop-aac.md) - the Rust interop publisher adds an AAC tone and every subscriber checks it beside Opus -- [Interop pre-skip](/quest/m1/interop-preskip.md) - the interop tone check anchors phase to timestamps, so a misapplied pre-skip fails - [Native audio decode conformance](/quest/m1/audio-decode-conformance.md) - Opus and AAC decode through every moq-audio backend, moq-c, and moq-ffi is checked against known tones and clips, like moq-video's suite - [moq-net internals](/quest/m1/net-internals.md) - no per-group session clones, no transport type parameter in lite and IETF internals, and the `Local` check measured - [Fragmented QUIC sends](/quest/m1/quic-send-fragments.md) - a chunk-size by retained-flight sweep decides whether `SendBuffer::get`'s front-to-back scan needs a cursor or index diff --git a/quest/m1/interop-aac.md b/quest/m1/interop-aac.md index 22b8edea50b..dc78d11c8d2 100644 --- a/quest/m1/interop-aac.md +++ b/quest/m1/interop-aac.md @@ -18,8 +18,8 @@ Things to look out for: each client decodes every rendition it is offered, or the harness asks for one by name per pass. - ffmpeg's AAC encoder adds priming samples; a decoder that ignores them shifts - the tone without a gap, which the check does not see (it does not anchor - phase to timestamps). + the tone without a gap, which `tone.py --phase` catches where `interop.sh`'s + `phase_checked` allows it. Check the AAC pairs it should cover actually pass. Public API: none expected. Wire: none. diff --git a/quest/m1/interop-preskip.md b/quest/m1/interop-preskip.md deleted file mode 100644 index c7a7f6df7b6..00000000000 --- a/quest/m1/interop-preskip.md +++ /dev/null @@ -1,34 +0,0 @@ -# [S] Interop tone check catches a misapplied pre-skip - -## Goal - -`test/interop/tone.py` fails a subscriber whose decoder drops or keeps the -wrong number of priming samples (Opus pre-skip, AAC priming), the way it -already fails a channel swap or a gap. Today a wrong pre-skip only shifts a -steady tone's phase, which the energy-and-frequency check cannot see. - -## Plan - -Found in review of the interop audio checks. The publishers' tone phase is a -function of the publisher's timestamps (the FFI clients and ffmpeg's `aevalsrc` -both start at zero), so the decoded sample at timestamp `t` should carry phase -`2πft`: anchor the Goertzel phase to the frame timestamps. - -Things to look out for: - -- Precision differs per client: the libav fallback stamps milliseconds (about - 1.4 rad at 440 Hz), and the browser publisher's capture clock is not tied to - its WAV's phase. Check only where the timestamps are precise enough, and say - which clients are left out. -- Add a negative control that misapplies the pre-skip on purpose, so the check - is proven able to fail. -- The GStreamer subscriber likely fails it today: `opusdec` trims pre-skip only - from a buffer's `GstAudioClippingMeta`, never from the OpusHead, and `moqsrc` - attaches none. - -Public API: none expected. Wire: none. - -## Related - -- [Native audio decode conformance](/quest/m1/audio-decode-conformance.md) - pre-skip continuity in unit tests -- [Interop AAC](/quest/m1/interop-aac.md) - AAC priming is the same question diff --git a/rs/moq-gst/Cargo.toml b/rs/moq-gst/Cargo.toml index 50156b4ea64..e0cc5ac763d 100644 --- a/rs/moq-gst/Cargo.toml +++ b/rs/moq-gst/Cargo.toml @@ -21,6 +21,7 @@ crate-type = ["cdylib", "rlib"] anyhow = { workspace = true, features = ["backtrace"] } bytes = { workspace = true } gst = { package = "gstreamer", version = "0.25" } +gst-audio = { package = "gstreamer-audio", version = "0.25" } hang = { workspace = true } moq-clock = { workspace = true } moq-mux = { workspace = true } diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 9316007c684..f0b6b376678 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1224,6 +1224,8 @@ impl Pump { } let mut sent_segment = false; + // Each pump starts a new stream on the pad, which opusdec decodes from a fresh state. + let mut pre_skip = PreSkip::new(&caps); loop { tokio::select! { // This rendition is being torn down (its run ended, or a catalog update replaced it). An @@ -1238,7 +1240,7 @@ impl Pump { frame = track.read() => match frame { Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { - let buffer = match build_buffer(frame, kind) { + let mut buffer = match build_buffer(frame, kind) { Ok(buffer) => buffer, Err(err) => { if let Some(obj) = element.upgrade() { @@ -1247,6 +1249,14 @@ impl Pump { return Some((kind, name)); } }; + if let Some(pre_skip) = &mut pre_skip + && let Err(err) = pre_skip.apply(buffer.get_mut().unwrap()) + { + if let Some(obj) = element.upgrade() { + gst::element_error!(obj, gst::StreamError::Format, ("track {name} has an invalid Opus packet"), ["{err:#}"]); + } + return Some((kind, name)); + } let segment = (!sent_segment).then(|| { let base = element.upgrade().and_then(|obj| obj.current_running_time()).unwrap_or(gst::ClockTime::ZERO); sent_segment = true; @@ -1336,6 +1346,52 @@ fn build_buffer(frame: moq_mux::container::Frame, kind: TrackKind) -> Result Option { + let structure = caps.structure(0).filter(|s| s.name() == "audio/x-opus")?; + let headers = structure.get_optional::("streamheader").ok()??; + let head = headers.first()?.get::().ok()?; + let head = head.map_readable().ok()?; + let total = moq_mux::codec::opus::Config::parse(&mut head.as_slice()).ok()?.pre_skip.into(); + (total > 0).then_some(Self { total, remaining: total }) + } + + /// Clip this packet's share of the pre-skip, and stamp it where its first kept sample plays. + /// A packet's timestamp is where its decode would start without the pre-skip, so each one plays + /// the pre-skip earlier, less what it clips itself. + fn apply(&mut self, buffer: &mut gst::BufferRef) -> Result<()> { + let mut clip = 0; + if self.remaining > 0 { + let samples = moq_mux::codec::opus::packet_samples(&buffer.map_readable()?).context("malformed Opus packet")?; + clip = self.remaining.min(samples.into()); + self.remaining -= clip; + gst_audio::AudioClippingMeta::add( + buffer, + gst::format::Default::from_u64(clip), + gst::format::Default::ZERO, + ); + } + let early = gst::ClockTime::from_nseconds( + (self.total - clip) + .mul_div_round(gst::ClockTime::SECOND.nseconds(), 48_000) + .context("pre-skip overflows the clock")?, + ); + // Only a packet clipped whole can reach before zero, and it plays nothing. + buffer.set_pts(buffer.pts().map(|pts| pts.saturating_sub(early))); + Ok(()) + } +} + fn video_caps(config: &hang::catalog::VideoConfig) -> Result { use hang::catalog::VideoCodec; @@ -1557,6 +1613,38 @@ mod tests { } } + /// opusdec only clips what a buffer's meta asks, one packet at a time, and stamps what's left + /// at the packet's pts, so the pre-skip spreads over as many packets as it spans and every + /// packet moves to where its first kept sample plays. + #[test] + fn opus_pre_skip_is_clipped_across_packets() { + gst::init().unwrap(); + let mut config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + assert!(super::PreSkip::new(&super::audio_caps(&config).unwrap()).is_none()); + config.description = Some(moq_mux::codec::opus::Config::new(48_000, 2).with_pre_skip(312).encode().unwrap()); + let caps = super::audio_caps(&config).unwrap(); + + // TOC configs 28 and 31: a 2.5 ms (120-sample) and a 20 ms (960-sample) CELT packet. + for (toc, micros, expected) in [ + (0xe0, 2_500, &[(120, 0), (120, 0), (72, 0), (0, 1_000), (0, 3_500)][..]), + (0xf8, 20_000, &[(312, 0), (0, 13_500)][..]), + ] { + let mut pre_skip = super::PreSkip::new(&caps).unwrap(); + for (index, &(clip, pts)) in expected.iter().enumerate() { + let mut buffer = gst::Buffer::from_slice([toc]); + let buffer = buffer.get_mut().unwrap(); + buffer.set_pts(gst::ClockTime::from_useconds(index as u64 * micros)); + pre_skip.apply(buffer).unwrap(); + let clipped = buffer.meta::().map(|meta| { + assert_eq!(meta.end(), gst::format::Default::ZERO.into()); + meta.start().value() as u64 + }); + assert_eq!(clipped.unwrap_or(0), clip, "packet {index} of {micros}us"); + assert_eq!(buffer.pts(), Some(gst::ClockTime::from_useconds(pts)), "packet {index} of {micros}us"); + } + } + } + // The shape type is generic, so the set math can be exercised with a plain integer standing // in for (caps, container): equal value == unchanged rendition, different value == reshape. fn renditions(pairs: &[(&str, u32)]) -> HashMap { diff --git a/rs/moq-mux/src/codec/opus/mod.rs b/rs/moq-mux/src/codec/opus/mod.rs index 408329b2ac5..a38d1772219 100644 --- a/rs/moq-mux/src/codec/opus/mod.rs +++ b/rs/moq-mux/src/codec/opus/mod.rs @@ -169,11 +169,10 @@ impl Config { /// Number of 48 kHz samples in an Opus packet, read from its TOC byte (RFC 6716 §3.1). /// -/// MPEG-TS aggregates several Opus packets into one PES, so the importer advances each -/// packet's timestamp by this. Opus timing is always reckoned at 48 kHz regardless of the -/// encoder's internal bandwidth. Returns `None` for an empty packet, a code-3 packet -/// missing its frame-count byte, or a packet outside the 1 frame and 120 ms RFC 6716 allows. -pub(crate) fn packet_samples(packet: &[u8]) -> Option { +/// Opus timing is always reckoned at 48 kHz regardless of the encoder's internal bandwidth. +/// Returns `None` for an empty packet, a code-3 packet missing its frame-count byte, or a +/// packet outside the 1 frame and 120 ms RFC 6716 allows. +pub fn packet_samples(packet: &[u8]) -> Option { let toc = *packet.first()?; let frames = match toc & 0b11 { 0 => 1, diff --git a/test/interop/README.md b/test/interop/README.md index ef8fc451977..43f172e1924 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -28,8 +28,12 @@ Go, and C++, `moq-c` for C, `decodebin` on `audio_0` for GStreamer, and 600 ms of PCM and frame timestamps for [`tone.py`](tone.py), which requires the publisher's channel count, contiguous timestamps, and each channel's tone carrying most of its energy by a Goertzel filter. A garbled decode (wrong channel order or -sample rate), silence, or a timestamp gap fails the cell; a misapplied pre-skip only -shifts the tone's phase, which this does not check. The browser +sample rate), silence, or a timestamp gap fails the cell. Each publisher's tone +starts at timestamp zero, so `--phase` also holds every tone to its timestamps +within a sample, which fails a decoder that keeps or drops the wrong number of +priming samples (Opus pre-skip). It skips the browser publisher, whose capture +clock is not tied to its WAV, and the native-JS subscriber, which leaves the +pre-skip to the player as WebCodecs does and stamps milliseconds. The browser runs the same filter on each channel of the player's audio graph. A negative control then has the Rust CLI swap its channels' tones, and every subscriber must refuse it as the wrong tone, not fail some other way. `interop.sh` is the one place the tones are chosen. diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 6743d6950b0..411f9e7d2c6 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -11,7 +11,8 @@ # # It stands up a moq-relay, then for each publisher language publishes an H.264 # broadcast with an Opus tone per audio channel, and confirms every subscriber sees -# video and decodes that tone, contiguous, before the timeout (tone.py judges the +# video and decodes that tone, contiguous and in phase with its timestamps (see +# phase_checked), before the timeout (tone.py judges the # decode; the browser judges its own playback). A publisher with its channels' # tones swapped must then fail every subscriber's check. The browser subscriber # also verifies rendered WebCodecs output and player pause/resume, then that a @@ -348,6 +349,14 @@ publisher_tone() { } # The wrong-tone control's publisher sends the stereo tone with its channels swapped. SWAPPED_TONE="660,440" +# phase_checked : whether tone.py also holds the tone's phase to the decode's +# timestamps, which fails a misapplied pre-skip. Every publisher's tone starts at timestamp zero +# except the browser's, whose capture clock is not tied to its WAV. The native-JS subscriber hands +# WebCodecs raw Opus without the OpusHead, as the player does, which keeps the pre-skip for the +# player to trim, and its libav fallback stamps milliseconds. +phase_checked() { + [[ "$1" != js* && "$2" != js-native-* ]] +} # Record a case for the CI report: `record case=C [field=value...]` (see # harness_result). Every session in a case dials the same relay with each client's own @@ -1263,7 +1272,9 @@ run_cell() { run_subscriber "$sub" "$broadcast" "$url" "$decoded" "$tone" || status=$? # The browser judges the tone in its page; every other subscriber leaves its decode here. if [[ "$status" -eq 0 && "$AUDIO" -eq 1 && "$sub" != js ]]; then - python3 "$INTEROP_DIR/tone.py" --tone "$tone" "$decoded" || status=$? + local phase=() + if phase_checked "$pub" "$sub"; then phase=(--phase); fi + python3 "$INTEROP_DIR/tone.py" --tone "$tone" ${phase[@]+"${phase[@]}"} "$decoded" || status=$? fi echo "$((SECONDS - started))" >"$LOGS/$pub-$sub.secs" return "$status" diff --git a/test/interop/tone.py b/test/interop/tone.py index c444803969d..82fcda25c93 100644 --- a/test/interop/tone.py +++ b/test/interop/tone.py @@ -8,10 +8,14 @@ This checks the layout matches the tone list, the frames are contiguous in time, and each channel carries its own tone, using a Goertzel filter at that channel's frequency. A garbled decode (wrong -channel order or sample rate), silence, or a timestamp gap fails. A misapplied pre-skip does -not: it only shifts a steady tone's phase, which this does not anchor to the timestamps. +channel order or sample rate), silence, or a timestamp gap fails. - python3 tone.py --tone 440,660 /run/rust-python-audio +Every publisher's tone starts at timestamp zero, so the sample decoded at timestamp t carries phase +2πft. With --phase, each channel's tone must hold that phase, which fails a decoder that keeps or +drops the wrong number of priming samples (Opus pre-skip, AAC priming): that only shifts a steady +tone in time. Pass it only where the publisher's and subscriber's timestamps are precise enough. + + python3 tone.py --tone 440,660 --phase /run/rust-python-audio Prints one `audio ok:` line, or `audio check failed: ` and exits 1. """ @@ -55,13 +59,28 @@ def goertzel_share(samples, rate, hz): return 2 * power / (len(samples) * energy) +def offset_us(samples, times, hz): + """How far ahead of its timestamps the tone `sin(2πft)` runs, in microseconds. + + Only defined modulo the tone's period, so the result is the nearest offset: within half a + period either way.""" + omega = 2 * math.pi * hz + re = im = 0.0 + for x, t in zip(samples, times): + re += x * math.cos(omega * t) + im -= x * math.sin(omega * t) + # sin(ω(t + d)) correlates with e^{-iωt} at phase ωd - π/2. + phase = math.remainder(math.atan2(im, re) + math.pi / 2, 2 * math.pi) + return phase / omega * 1_000_000 + + def dominant_hz(samples, rate): """The strongest frequency in 20 Hz steps, to say what a failing channel carries instead.""" probe = samples[: rate // 10] return max(range(100, 4000, 20), key=lambda hz: goertzel_share(probe, rate, hz)) -def check(prefix, tones): +def check(prefix, tones, phase=False): lines = Path(f"{prefix}.txt").read_text().split("\n") try: rate, channels = (int(v) for v in lines[0].split()) @@ -88,7 +107,9 @@ def check(prefix, tones): raise Failure(f"only {total * 1000 // rate}ms decoded, need {SKIP_MS + WINDOW_MS}ms") samples = struct.unpack(f"<{total * channels}f", pcm) - shares = [] + # Each judged sample's time, for --phase: its frame's timestamp plus its place in the frame. + times = [ts / 1_000_000 + n / rate for ts, count in frames for n in range(count)][skip:] if phase else None + judged = [] for channel, hz in enumerate(tones): data = samples[skip * channels + channel :: channels] rms = math.sqrt(sum(x * x for x in data) / len(data)) @@ -100,21 +121,33 @@ def check(prefix, tones): f"wrong tone: channel {channel} carries ~{dominant_hz(data, rate)} Hz, not {hz} Hz " f"({share:.2f} of its energy at {hz} Hz)" ) - shares.append((share, rms)) + result = f"{hz} Hz {share:.2f} (rms {rms:.2f}" + if times: + offset = offset_us(data, times, hz) + # Within a sample: a clean decode measures a few microseconds off, while Opus's usual + # 312-sample pre-skip, kept or trimmed twice, moves a 440 Hz tone 318us. Only a miscount + # of whole periods (2273us at 440 Hz, 4545us across 440 and 660 Hz) would hide. + if abs(offset) > 1_000_000 / rate: + raise Failure( + f"tone off its timestamps: channel {channel}'s {hz} Hz is {offset:+.0f}us out " + f"(a misapplied pre-skip or priming?)" + ) + result += f", {offset:+.0f}us" + judged.append(result + ")") window = (total - skip) * 1000 // rate - summary = ", ".join(f"{hz} Hz {share:.2f} (rms {rms:.2f})" for hz, (share, rms) in zip(tones, shares)) - return f"{channels} ch @ {rate} Hz, {window}ms judged, {len(frames)} contiguous frames: {summary}" + return f"{channels} ch @ {rate} Hz, {window}ms judged, {len(frames)} contiguous frames: {', '.join(judged)}" def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--tone", required=True, help="comma-separated frequency of each channel, in Hz") + parser.add_argument("--phase", action="store_true", help="also require each tone's phase to match the timestamps") parser.add_argument("prefix", help="the subscriber's --audio-out prefix") args = parser.parse_args() tones = [int(hz) for hz in args.tone.split(",")] try: - print(f"audio ok: {check(args.prefix, tones)}") + print(f"audio ok: {check(args.prefix, tones, args.phase)}") except (Failure, OSError) as err: print(f"audio check failed: {err}") sys.exit(1) diff --git a/test/interop/tone.test.py b/test/interop/tone.test.py index 0a8b0a2345b..bbc35409882 100644 --- a/test/interop/tone.test.py +++ b/test/interop/tone.test.py @@ -18,8 +18,9 @@ def setUp(self): self.addCleanup(temp.cleanup) self.prefix = Path(temp.name) / 'audio' - def write(self, tones, *, rate=48000, generated_at=None, frame=960, frames=40, gap_after=None, start=123_456): - """Decode output as a subscriber writes it: `frames` frames of `frame` samples per channel.""" + def write(self, tones, *, rate=48000, generated_at=None, frame=960, frames=40, gap_after=None, start=123_456, late=0): + """Decode output as a subscriber writes it: `frames` frames of `frame` samples per channel, + each tone `sin(2πft)` at its timestamp, or `late` samples behind it.""" generated_at = generated_at or rate log = [f'{rate} {len(tones)}'] pcm = bytearray() @@ -27,7 +28,7 @@ def write(self, tones, *, rate=48000, generated_at=None, frame=960, frames=40, g for index in range(frames): log.append(f'{ts} {frame}') for n in range(index * frame, (index + 1) * frame): - pcm += struct.pack(f'<{len(tones)}f', *(0.5 * math.sin(2 * math.pi * hz * n / generated_at) for hz in tones)) + pcm += struct.pack(f'<{len(tones)}f', *(0.5 * math.sin(2 * math.pi * hz * (start / 1_000_000 + (n - late) / generated_at)) for hz in tones)) ts += frame * 1_000_000 // rate if index == gap_after: ts += 20_000 @@ -36,16 +37,16 @@ def write(self, tones, *, rate=48000, generated_at=None, frame=960, frames=40, g def assertFails(self, tones, reason): with self.assertRaises(tone.Failure) as caught: - tone.check(self.prefix, tones) + tone.check(self.prefix, tones, phase=True) self.assertIn(reason, str(caught.exception)) def test_a_clean_stereo_decode_passes(self): self.write([440, 660]) - self.assertIn('2 ch @ 48000 Hz', tone.check(self.prefix, [440, 660])) + self.assertIn('2 ch @ 48000 Hz', tone.check(self.prefix, [440, 660], phase=True)) def test_a_clean_mono_decode_passes(self): self.write([440], frame=120, frames=320) - self.assertIn('1 ch', tone.check(self.prefix, [440])) + self.assertIn('1 ch', tone.check(self.prefix, [440], phase=True)) def test_swapped_channels_fail(self): self.write([660, 440]) @@ -71,6 +72,20 @@ def test_a_timestamp_gap_fails(self): self.write([440], gap_after=10) self.assertFails([440], 'timestamp gap') + # Negative controls for --phase: a decoder that keeps Opus's usual 312 samples of pre-skip, or + # trims them twice, decodes a clean tone that only its timestamps can show is misplaced. + def test_a_kept_pre_skip_fails(self): + self.write([440, 660], late=312) + self.assertFails([440, 660], "tone off its timestamps: channel 0's 440 Hz is +318us") + + def test_a_pre_skip_trimmed_twice_fails(self): + self.write([440], frame=120, frames=320, late=-312) + self.assertFails([440], 'is -318us') + + def test_a_kept_pre_skip_passes_without_phase(self): + self.write([440, 660], late=312) + self.assertIn('2 ch', tone.check(self.prefix, [440, 660])) + def test_too_little_audio_fails(self): self.write([440], frames=10) self.assertFails([440], 'only 200ms decoded') From 0f72ce94720eb88ae637f031585083a5f50760bb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 00:08:30 -0700 Subject: [PATCH 3/8] style(gst): rustfmt the pre-skip test Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/source/imp.rs | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index f0b6b376678..1eea92a45b0 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1362,8 +1362,14 @@ impl PreSkip { let headers = structure.get_optional::("streamheader").ok()??; let head = headers.first()?.get::().ok()?; let head = head.map_readable().ok()?; - let total = moq_mux::codec::opus::Config::parse(&mut head.as_slice()).ok()?.pre_skip.into(); - (total > 0).then_some(Self { total, remaining: total }) + let total = moq_mux::codec::opus::Config::parse(&mut head.as_slice()) + .ok()? + .pre_skip + .into(); + (total > 0).then_some(Self { + total, + remaining: total, + }) } /// Clip this packet's share of the pre-skip, and stamp it where its first kept sample plays. @@ -1372,14 +1378,11 @@ impl PreSkip { fn apply(&mut self, buffer: &mut gst::BufferRef) -> Result<()> { let mut clip = 0; if self.remaining > 0 { - let samples = moq_mux::codec::opus::packet_samples(&buffer.map_readable()?).context("malformed Opus packet")?; + let samples = + moq_mux::codec::opus::packet_samples(&buffer.map_readable()?).context("malformed Opus packet")?; clip = self.remaining.min(samples.into()); self.remaining -= clip; - gst_audio::AudioClippingMeta::add( - buffer, - gst::format::Default::from_u64(clip), - gst::format::Default::ZERO, - ); + gst_audio::AudioClippingMeta::add(buffer, gst::format::Default::from_u64(clip), gst::format::Default::ZERO); } let early = gst::ClockTime::from_nseconds( (self.total - clip) @@ -1621,7 +1624,12 @@ mod tests { gst::init().unwrap(); let mut config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); assert!(super::PreSkip::new(&super::audio_caps(&config).unwrap()).is_none()); - config.description = Some(moq_mux::codec::opus::Config::new(48_000, 2).with_pre_skip(312).encode().unwrap()); + config.description = Some( + moq_mux::codec::opus::Config::new(48_000, 2) + .with_pre_skip(312) + .encode() + .unwrap(), + ); let caps = super::audio_caps(&config).unwrap(); // TOC configs 28 and 31: a 2.5 ms (120-sample) and a 20 ms (960-sample) CELT packet. @@ -1640,7 +1648,11 @@ mod tests { meta.start().value() as u64 }); assert_eq!(clipped.unwrap_or(0), clip, "packet {index} of {micros}us"); - assert_eq!(buffer.pts(), Some(gst::ClockTime::from_useconds(pts)), "packet {index} of {micros}us"); + assert_eq!( + buffer.pts(), + Some(gst::ClockTime::from_useconds(pts)), + "packet {index} of {micros}us" + ); } } } From 623de58887bf0220dd1c95f8d87d4950fcd0edf5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 12:37:46 -0700 Subject: [PATCH 4/8] test(interop): mark phase_checked as reached through run_cell Merging main left phase_checked called only from run_cell, which harness_spawn invokes indirectly, so shellcheck flagged it as unused. Co-Authored-By: Claude Opus 5.5 --- test/interop/interop.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 45338bcabf0..5154f4d937b 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -355,6 +355,7 @@ SWAPPED_TONE="660,440" # except the browser's, whose capture clock is not tied to its WAV. The native-JS subscriber hands # WebCodecs raw Opus without the OpusHead, as the player does, which keeps the pre-skip for the # player to trim, and its libav fallback stamps milliseconds. +# shellcheck disable=SC2329 # reached from run_cell, which 'harness_spawn' invokes phase_checked() { [[ "$1" != js* && "$2" != js-native-* ]] } From 5201a4a13bf2141653aff2042194c62e30c41b83 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 13:38:05 -0700 Subject: [PATCH 5/8] doc(gst): describe moqsrc's Opus pre-skip clipping Co-Authored-By: Claude Opus 5.5 --- doc/bin/gstreamer.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index b77e58889fb..6a7c321e752 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -134,7 +134,10 @@ every returning rendition; one it omits gets EOS instead of keeping its old pad. Within each run, the first media frame fixes one timestamp reference and segment base for every pad. Audio and video keep their relative broadcast timing, even when their tracks use different timescales or arrive at different times. Frames -before that reference keep an earlier PTS for downstream to clip. A new broadcast +before that reference keep an earlier PTS for downstream to clip. Opus packets +covering the OpusHead pre-skip carry `GstAudioClippingMeta` for it, and each +Opus PTS marks where its first kept sample plays, so `opusdec` drops the +priming samples and the audio lines up with video. A new broadcast chooses a fresh reference on the same pads; playback does not require a wall clock. A rendition that leaves the catalog gets EOS once its track ends. A broadcast that ends or loses its publisher holds From 5b78faa88934e3f6da150049a9b0e0a766c6e965 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 16:49:08 -0700 Subject: [PATCH 6/8] doc(hang): state that Opus frame timestamps count the pre-skip Co-Authored-By: Claude Opus 5.5 --- doc/bin/gstreamer.md | 4 ++-- drafts/draft-lcurley-moq-hang.md | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index 6a7c321e752..98ed0d14708 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -136,8 +136,8 @@ base for every pad. Audio and video keep their relative broadcast timing, even when their tracks use different timescales or arrive at different times. Frames before that reference keep an earlier PTS for downstream to clip. Opus packets covering the OpusHead pre-skip carry `GstAudioClippingMeta` for it, and each -Opus PTS marks where its first kept sample plays, so `opusdec` drops the -priming samples and the audio lines up with video. A new broadcast +Opus packet that keeps samples is stamped where the first one plays, so `opusdec` +drops the priming samples and the audio lines up with video. A new broadcast chooses a fresh reference on the same pads; playback does not require a wall clock. A rendition that leaves the catalog gets EOS once its track ends. A broadcast that ends or loses its publisher holds diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index fe79214a0f0..f99d0120085 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -412,6 +412,8 @@ type AudioDecoderConfigExtensions = { A consumer MUST reject `ambisonic` with fewer than 4 channels or a count that is not a square, and MUST treat an unrecognized `layout` as absent. `layout` is not part of the track's identity. +An Opus frame timestamp counts the samples the OpusHead pre-skip trims: a consumer presents each decoded sample the pre-skip, at 48 kHz, earlier than its frame timestamp places it, so the first sample kept plays at the first decoded frame's timestamp. + ### PCM {#audio-pcm} Hang defines the `"pcm"` audio codec for uncompressed samples. @@ -1201,6 +1203,7 @@ This document has no IANA actions. - Added an optional `epoch` beside every rendition and data track `broadcast` reference, and `replayEpoch` beside the archive's `replay` ({{field-epoch}}). A consumer reads each referenced path from the first broadcast it resolves for a catalog instance; a present `epoch` fails a rendition proven to resolve another instance, and is best effort over a route without an Epoch. - Derived HLS and DASH exports hand disabled references to enabled renditions at the first record numbered past the advertised ones, keeping the record `sequence` as the HLS media sequence and starting a DASH period. - Added optional `stats` and `echo` catalog sections, publisher/viewer snapshots keyed by rendition ID, and the `.echo` feedback broadcast convention. +- An Opus frame timestamp counts the OpusHead pre-skip: a consumer presents each decoded sample the pre-skip earlier than its frame timestamp places it. - Fixed a track's identity for its name ({{identity}}): codec, description, container, display aspect, sample rate, and channel count never change, while every other field, including size, bitrate, and framerate, updates in place. A change of identity takes a new track name, which a rendition may name with `track` while keeping its ID, or a new Epoch. - Specified MSF text metadata preservation and refusal of unrepresentable text packaging during gateway conversion. From df1c81eb2d166ef941f80cee43ab04fad74e90b6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 01:49:42 -0700 Subject: [PATCH 7/8] fix(gst): moqsink restores the Opus pre-skip; phase-check Opus only opusenc and moqsrc stamp an Opus packet where its first kept sample plays and clip the pre-skip with GstAudioClippingMeta, while a hang timestamp counts the pre-skip. moqsink published the GStreamer stamp, so packets overlapped by the pre-skip and a moqsrc to moqsink hop shifted twice. The AAC rendition's tone is 1024 priming samples late because the fMP4 importer ignores the edit list ffmpeg writes; that is quest/m1/fmp4-edit-lists.md, so --phase checks Opus until it lands. Co-Authored-By: Claude Opus 5.5 --- doc/bin/gstreamer.md | 4 + quest/m1/README.md | 1 + quest/{m2 => m1}/fmp4-edit-lists.md | 20 ++++- quest/m2/README.md | 1 - rs/moq-gst/src/sink/imp.rs | 2 +- rs/moq-gst/src/sink/pad.rs | 128 +++++++++++++++++++++++++--- test/interop/README.md | 21 ++--- test/interop/interop.sh | 14 +-- 8 files changed, 159 insertions(+), 32 deletions(-) rename quest/{m2 => m1}/fmp4-edit-lists.md (55%) diff --git a/doc/bin/gstreamer.md b/doc/bin/gstreamer.md index 98ed0d14708..161d462bd2a 100644 --- a/doc/bin/gstreamer.md +++ b/doc/bin/gstreamer.md @@ -49,6 +49,10 @@ and stereo use a family 0 head, and three to eight channels need `channel-mapping-family`, `stream-count`, `coupled-count`, and `channel-mapping`. Caps that omit that mapping, or that contradict the header, are refused. `opusenc` supplies both, so a 5.1 encode plays as six channels. +Opus buffers are read the way `opusenc` and `moqsrc` stamp them: at the time +their first kept sample plays, with `GstAudioClippingMeta` clipping the +pre-skip. Each is published the pre-skip, less its own clipping, later, since +a frame timestamp counts the pre-skip. A `text/x-raw` pad is how captions get in: ffmpeg cannot mux a subtitle track into fragmented MP4, so `moq import fmp4` can't carry one, while a demuxer that diff --git a/quest/m1/README.md b/quest/m1/README.md index ddc1b4c7050..791792a4b5f 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -118,6 +118,7 @@ audit's receive-memory, priority, and lease bugs rank ahead of that reshape. - [More tests under load](/quest/m1/test-flakes-2/README.md) - the second round of load-only failures, one quest per flake, fixed at the cause - [Broadcast lifecycle](/quest/m1/broadcast-lifecycle/README.md) - broadcasts are online or offline with no finished state in moq-net, finished-broadcast retention is a fixed moq-mux policy, and each linger timer has one meaning - [JS interop tone](/quest/m1/interop-js-tone.md) - the `js ->` cells pass the audio check on every draft again, after the 2026-10-23 regression +- [fMP4 edit lists](/quest/m1/fmp4-edit-lists.md) - the fMP4 importer applies edit lists to frame timestamps, so the interop AAC tone passes the phase check - [A smaller, safer API than release](/quest/m1/api-audit/README.md) - every surface added since release has a consumer or is gone, overlapping operations are collapsed, names mirror across Rust, JS, and the bindings, and every break says how to migrate - [moq-quic's API improves on quinn's](/quest/m1/quic-api/README.md) - moq-quic and both QUIC drivers ship one mirrored API: `send::`/`recv::` namespaces, one error per direction, consuming terminal operations, plain configs, no quinn baggage - [JavaScript FETCH](/quest/m1/js-fetch.md) - browser publishers answer IETF FETCH through the JS ranges request surface diff --git a/quest/m2/fmp4-edit-lists.md b/quest/m1/fmp4-edit-lists.md similarity index 55% rename from quest/m2/fmp4-edit-lists.md rename to quest/m1/fmp4-edit-lists.md index 931ae0e605e..57853124904 100644 --- a/quest/m2/fmp4-edit-lists.md +++ b/quest/m1/fmp4-edit-lists.md @@ -4,23 +4,35 @@ An fMP4 source with an edit list (AAC priming, a non-zero `media_time`, or an empty edit) publishes frames at their presentation time, so both players and -every export see the timeline the source meant. +every export see the timeline the source meant. The interop AAC rendition +then passes `tone.py --phase` like Opus does. ## Plan Today `edts` is forwarded in the catalog init but read by no Rust or JS code, and `rs/moq-mux/src/container/fmp4/export.rs` strips it on the way out. Both players present `tfdt` plus the composition offset. ffmpeg's fragmented -output avoids `elst` by default, so this mostly affects GPAC, Bento4, and -Shaka sources (2026-10-07 audit). +output does write `elst`: with the interop's `-movflags cmaf`, its AAC track +carries `media_time` 1024 at `tfdt` 0, so every subscriber plays the AAC tone +1024 samples late (`-880us` at 440 Hz once `--phase` checks it), and its Opus +track carries `media_time` equal to the `dOps` pre-skip. Decided (2026-10-07): the importer applies the edit list to each frame's timestamp and drops `edts` from the catalog init, so no consumer needs edit list code. This builds on decoders taking time from the frame timestamp rather than `tfdt`. Decide in the PR whether priming samples before the edit -are dropped or published with negative-offset handling, and refuse edit +are dropped or published with negative-offset handling (Apple's 2112-sample +AAC priming is not a whole number of 1024-sample frames), and refuse edit lists with more than one non-empty entry rather than guess. +Opus frame timestamps count the OpusHead pre-skip (`draft-lcurley-moq-hang`), +so for Opus the importer applies only the part of the edit beyond the +pre-skip, or the pre-skip is trimmed twice. + +Once AAC lines up, drop the `opus` condition from `phase_checked` in +`test/interop/interop.sh` and the matching sentence in +`test/interop/README.md`. + Public API: none. Wire: none, but a behavior change: frame timestamps now include the edit and the init loses `edts`, so a consumer that offsets AAC priming on its own must stop, or it shifts twice. Note it in the changelog. diff --git a/quest/m2/README.md b/quest/m2/README.md index c61eef15bf6..09f47af2d50 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -127,7 +127,6 @@ lite-07; bitrate enforcement is not part of this release. - [Drop announce="never"](/quest/m2/publish-announce-never.md) - `@moq/publish` offers only announce modes a watcher can reach - [moqsrc A/V sync](/quest/m2/moqsrc-av-sync.md) - a nightly test measures A/V skew from `moqsrc` through clock-synced sinks - [MP4 export](/quest/m2/mp4-export.md) - `moq export mp4 --output` records crash-safe fragments, then finishes a regular MP4 with moov at the end -- [fMP4 edit lists](/quest/m2/fmp4-edit-lists.md) - the fMP4 importer applies edit lists to frame timestamps - [Importers fill the VR fields](/quest/m2/mux-vr-fields.md) - MP4, Matroska, and Opus imports set the catalog's projection, stereo, and ambisonic layout - [Linux decoded frames](/quest/m2/obs-decode-linux.md) - present supported native decoded surfaces with visible CPU fallback - [Windows decoded frames](/quest/m2/obs-decode-windows.md) - present decoded D3D11 surfaces in OBS without CPU readback diff --git a/rs/moq-gst/src/sink/imp.rs b/rs/moq-gst/src/sink/imp.rs index 7e876a8436d..dc622229c82 100644 --- a/rs/moq-gst/src/sink/imp.rs +++ b/rs/moq-gst/src/sink/imp.rs @@ -715,7 +715,6 @@ impl MoqSink { // Map and copy outside the lock so the per-pad lock covers only the producer write. Avoiding the // copy for an oversized buffer needs a reliable, media-aware size heuristic; moq-net remains the // authority and rejects FrameTooLarge before reserving its own group slot. - let pts = buffer.pts(); // Only subtitles use this: a cue needs an explicit end, unlike a media frame. let duration = buffer.duration(); let current_running_time = self.obj().current_running_time(); @@ -756,6 +755,7 @@ impl MoqSink { return Ok(gst::FlowSuccess::Ok); } let outcome = lifecycle.media.place_wall(&buffer, sample).and_then(|()| { + let pts = lifecycle.media.timeline_pts(&buffer)?; lifecycle .media .push_buffer(data, pts, duration, current_running_time, moq_net::time::Instant::now()) diff --git a/rs/moq-gst/src/sink/pad.rs b/rs/moq-gst/src/sink/pad.rs index e459c494880..21dc72a577a 100644 --- a/rs/moq-gst/src/sink/pad.rs +++ b/rs/moq-gst/src/sink/pad.rs @@ -61,6 +61,8 @@ struct Media { /// a successful decode may publish nothing (a header-only buffer), and a delta only misses its /// keyframe while no group is open. keyframe: bool, + /// An Opus pad's pre-skip, which moves its PTS onto hang's timeline. + pre_skip: Option, } impl Media { @@ -410,6 +412,7 @@ impl Pad { return Ok(name); } + let mut pre_skip = None; let (track, audio, name): (import::Track, bool, String) = match structure.name().as_str() { "video/x-h264" => { let (track, name) = Self::reserve_video( @@ -513,6 +516,7 @@ impl Pad { // The OpusHead lives in the caps. There is no init buffer in the stream, so a // head built from channels and rate alone would drop a surround mapping. let config = opus_catalog(structure)?; + pre_skip = PreSkip::new(&config)?; let name = Self::track_name(&broadcast, requested, ".opus"); let request = broadcast .reserve_track(name.clone()) @@ -534,6 +538,7 @@ impl Pad { encoder, discontinuity: false, keyframe: !audio, + pre_skip, }))); self.caps = Some(caps.clone()); Ok(name) @@ -750,6 +755,18 @@ impl Pad { /// (or an oversized frame, rejected by moq-net) invalidates only this pad and says so in the /// returned outcome. Returns an error when an unstamped opaque buffer has no current /// running time, so the caller fails the flow instead of silently dropping data. + /// The PTS `buffer` publishes at: its own, except an Opus packet's, which moves onto hang's + /// timeline (see [`PreSkip`]). + pub(super) fn timeline_pts(&mut self, buffer: &gst::BufferRef) -> Result> { + match &mut self.track { + Some(Sink::Media(media)) if !self.failed => match &mut media.pre_skip { + Some(pre_skip) => pre_skip.pts(buffer), + None => Ok(buffer.pts()), + }, + _ => Ok(buffer.pts()), + } + } + pub fn push_buffer( &mut self, data: Bytes, @@ -907,6 +924,67 @@ fn opus_catalog(structure: &gst::StructureRef) -> Result Result> { + let Some(head) = &config.description else { + return Ok(None); + }; + let total = moq_mux::codec::opus::Config::parse(&mut head.as_ref())?.pre_skip.into(); + Ok((total > 0).then_some(Self { total, clipped: 0 })) + } + + /// The hang timestamp of `buffer`, an Opus packet stamped the GStreamer way. + fn pts(&mut self, buffer: &gst::BufferRef) -> Result> { + let Some(pts) = buffer.pts() else { + return Ok(None); + }; + let clip = match buffer.meta::().map(|meta| meta.start()) { + None => 0, + Some(gst::GenericFormattedValue::Default(clip)) => clip.map_or(0, |clip| *clip), + Some(other) => anyhow::bail!("Opus clipping meta counts {:?}, not samples", other.format()), + }; + let late = self + .total + .checked_sub(clip) + .context("Opus clipping meta trims more than the OpusHead pre-skip")?; + if clip > 0 && pts.is_zero() { + let samples: u64 = moq_mux::codec::opus::packet_samples(&buffer.map_readable()?) + .context("malformed Opus packet")? + .into(); + if clip >= samples { + // A packet clipped whole plays nothing, so its stamp saturated at zero and lost where + // it sat: place it after those before it, as a stream starting at zero has them. + let at = self.clipped; + self.clipped += samples; + return Ok(Some(samples_time(at)?)); + } + } + self.clipped = 0; + Ok(Some(pts + samples_time(late)?)) + } +} + +/// The duration of `samples` at Opus's 48 kHz. +fn samples_time(samples: u64) -> Result { + use gst::prelude::MulDiv; + let nanos = samples + .mul_div_round(gst::ClockTime::SECOND.nseconds(), 48_000) + .context("Opus pre-skip overflows the clock")?; + Ok(gst::ClockTime::from_nseconds(nanos)) +} + fn opus_count(structure: &gst::StructureRef, field: &str, what: &str) -> Result { let value: i32 = structure .get(field) @@ -1450,14 +1528,26 @@ mod tests { /// `opusenc` 5.1 caps, plus the packets it produced. fn opusenc_surround() -> (gst::Caps, Vec) { + let (caps, buffers) = opusenc( + "audio/x-raw,format=S16LE,layout=interleaved,rate=48000,channels=6,channel-mask=(bitmask)0x3f", + "20", + ); + let packets = buffers + .iter() + .map(|buffer| Bytes::copy_from_slice(buffer.map_readable().expect("map opus packet").as_slice())) + .collect(); + (caps, packets) + } + + /// Three 20 ms buffers of `raw` through `opusenc frame-size=`: its caps, plus the + /// packets it produced. + fn opusenc(raw: &str, frame_size: &str) -> (gst::Caps, Vec) { use gst::prelude::*; use std::sync::{Arc, Mutex}; - let pipeline = gst::parse::launch( - "audiotestsrc num-buffers=3 samplesperbuffer=960 ! \ - audio/x-raw,format=S16LE,layout=interleaved,rate=48000,channels=6,channel-mask=(bitmask)0x3f ! \ - opusenc name=enc ! fakesink", - ) + let pipeline = gst::parse::launch(&format!( + "audiotestsrc num-buffers=3 samplesperbuffer=960 ! {raw} ! opusenc name=enc frame-size={frame_size} ! fakesink" + )) .expect("opusenc pipeline") .downcast::() .expect("pipeline"); @@ -1476,11 +1566,7 @@ mod tests { *caps_probe.lock().unwrap() = pad.current_caps(); } if let Some(buffer) = info.buffer() { - let map = buffer.map_readable().expect("map opus packet"); - packets_probe - .lock() - .unwrap() - .push(Bytes::copy_from_slice(map.as_slice())); + packets_probe.lock().unwrap().push(buffer.copy()); } gst::PadProbeReturn::Ok }) @@ -1511,6 +1597,28 @@ mod tests { (caps, packets) } + /// opusenc stamps each packet where its first kept sample plays, saturating at zero, and clips + /// the pre-skip; restoring hang's timeline lays its packets back to back from zero, both when + /// the pre-skip fits in the first packet and when it clips several whole. + #[test] + fn opusenc_packets_land_back_to_back_on_hang_timeline() { + gst::init().unwrap(); + for (frame_size, micros) in [("20", 20_000), ("2.5", 2_500)] { + let (caps, buffers) = opusenc("audio/x-raw,rate=48000,channels=1", frame_size); + let config = opus_catalog(caps.structure(0).unwrap()).unwrap(); + let mut pre_skip = PreSkip::new(&config).unwrap().expect("opusenc sets a pre-skip"); + assert!(buffers[0].meta::().is_some()); + for (index, buffer) in buffers.iter().enumerate() { + assert_eq!( + pre_skip.pts(buffer).unwrap(), + Some(gst::ClockTime::from_useconds(index as u64 * micros)), + "packet {index} of {frame_size} ms, stamped {:?}", + buffer.pts() + ); + } + } + } + // The regression: a real 5.1 opusenc stream publishes the family 1 head its caps // carry, and moq-audio decodes that head to six channels. #[test] diff --git a/test/interop/README.md b/test/interop/README.md index f4edfd80844..74dbb10a68d 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -32,17 +32,18 @@ Go, and C++, `moq-c` for C, `decodebin` on each `audio_N` pad for GStreamer, and publisher's channel count, contiguous timestamps, and each channel's tone carrying most of its energy by a Goertzel filter. A garbled decode (wrong channel order or sample rate), silence, a timestamp gap, or a rendition left undecoded fails the cell. -Each publisher's tone starts at timestamp zero, so `--phase` also holds every +Each publisher's tone starts at timestamp zero, so `--phase` also holds every Opus rendition's tone to its timestamps within a sample, which fails a decoder that keeps -or drops the wrong number of priming samples (Opus pre-skip or AAC priming). It skips -the browser publisher, whose capture clock is not tied to its WAV, and the native-JS -subscriber, which leaves the pre-skip to the player as WebCodecs does and stamps -milliseconds. The browser plays each rendition in a page of its own, the player pinned -to it by `controls.audio.target`, and runs the same filter on each channel of the -player's audio graph. A negative control then has the Rust CLI swap its channels' -tones, and every subscriber must refuse every rendition as the wrong tone, not fail -some other way. `interop.sh` is the one place the tones and each publisher's codecs -are chosen. +or drops the wrong pre-skip. It skips the browser publisher, whose capture clock is +not tied to its WAV, and the native-JS subscriber, which leaves the pre-skip to the +player as WebCodecs does and stamps milliseconds. AAC is skipped until the fMP4 +importer applies the edit list holding ffmpeg's priming +([quest](/quest/m1/fmp4-edit-lists.md)). The browser plays each rendition in a page +of its own, the player pinned to it by `controls.audio.target`, and runs the same +filter on each channel of the player's audio graph. A negative control then has the +Rust CLI swap its channels' tones, and every subscriber must refuse every rendition +as the wrong tone, not fail some other way. `interop.sh` is the one place the tones +and each publisher's codecs are chosen. The normal run also checks a finite raw track through the relay. Its publisher announces end 4 before writing four 256 KiB groups; the reader verifies every byte, diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 4da2ac671fa..9f8bd217227 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -360,14 +360,16 @@ publisher_codecs() { *) echo opus ;; esac } -# phase_checked : whether tone.py also holds the tone's phase to the decode's -# timestamps, which fails a misapplied pre-skip or AAC priming. Every publisher's tone starts at +# phase_checked : whether tone.py also holds the tone's phase to the +# decode's timestamps, which fails a misapplied pre-skip. Every publisher's tone starts at # timestamp zero except the browser's, whose capture clock is not tied to its WAV. The native-JS # subscriber hands WebCodecs raw Opus without the OpusHead, as the player does, which keeps the -# pre-skip for the player to trim, and its libav fallback stamps milliseconds. +# pre-skip for the player to trim, and its libav fallback stamps milliseconds. AAC waits on the +# fMP4 importer applying the edit list that carries ffmpeg's 1024 priming samples +# (quest/m1/fmp4-edit-lists.md). # shellcheck disable=SC2329 # reached from run_cell, which 'harness_spawn' invokes phase_checked() { - [[ "$1" != js* && "$2" != js-native-* ]] + [[ "$1" != js* && "$2" != js-native-* && "$3" == opus ]] } # Record a case for the CI report: `record case=C [field=value...]` (see @@ -1392,9 +1394,9 @@ run_cell() { # rendition here, and a missing one fails like a garbled one. Each is judged, so the # wrong-tone control can require every rendition to fail. if [[ "$status" -eq 0 && "$AUDIO" -eq 1 && "$sub" != js ]]; then - local phase=() - if phase_checked "$pub" "$sub"; then phase=(--phase); fi for codec in ${codecs//,/ }; do + local phase=() + if phase_checked "$pub" "$sub" "$codec"; then phase=(--phase); fi echo "the $codec rendition:" python3 "$INTEROP_DIR/tone.py" --tone "$tone" ${phase[@]+"${phase[@]}"} "$decoded.$codec" || status=$? done From 3d8d8d915d2c1399b6d559a6dd6a7e0f5a070641 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 02:28:14 -0700 Subject: [PATCH 8/8] fix(gst): moqsink shifts Opus clipping past the pre-skip earlier GstAudioClippingMeta can clip more than the OpusHead pre-skip (a seek or an edit). Hang has no way to drop those samples, so land the packet earlier and let its kept samples play at its PTS, instead of failing the pad. Only a shift before zero is refused. Co-Authored-By: Claude Opus 5.5 --- rs/moq-gst/src/sink/pad.rs | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/rs/moq-gst/src/sink/pad.rs b/rs/moq-gst/src/sink/pad.rs index 21dc72a577a..571f1bcb439 100644 --- a/rs/moq-gst/src/sink/pad.rs +++ b/rs/moq-gst/src/sink/pad.rs @@ -955,10 +955,6 @@ impl PreSkip { Some(gst::GenericFormattedValue::Default(clip)) => clip.map_or(0, |clip| *clip), Some(other) => anyhow::bail!("Opus clipping meta counts {:?}, not samples", other.format()), }; - let late = self - .total - .checked_sub(clip) - .context("Opus clipping meta trims more than the OpusHead pre-skip")?; if clip > 0 && pts.is_zero() { let samples: u64 = moq_mux::codec::opus::packet_samples(&buffer.map_readable()?) .context("malformed Opus packet")? @@ -972,7 +968,15 @@ impl PreSkip { } } self.clipped = 0; - Ok(Some(pts + samples_time(late)?)) + // Clipping beyond the pre-skip (a seek, an edit) has no hang equivalent: those samples still + // play, and the packet lands earlier so its kept ones play where GStreamer put them. + match self.total.checked_sub(clip) { + Some(late) => Ok(Some(pts + samples_time(late)?)), + None => pts + .checked_sub(samples_time(clip - self.total)?) + .context("Opus clipping meta reaches before zero") + .map(Some), + } } } @@ -1616,6 +1620,26 @@ mod tests { buffer.pts() ); } + + // Clipping past the pre-skip (a seek, an edit) lands the packet earlier instead, so its + // kept samples still play at its PTS, and refuses to reach before zero. + let clip = pre_skip.total + 48; + for (pts, expected) in [(10_000, Some(9_000)), (500, None)] { + let mut buffer = gst::Buffer::from_slice([0xf8]); + let buffer_mut = buffer.get_mut().unwrap(); + buffer_mut.set_pts(gst::ClockTime::from_useconds(pts)); + gst_audio::AudioClippingMeta::add( + buffer_mut, + gst::format::Default::from_u64(clip), + gst::format::Default::ZERO, + ); + let restored = pre_skip.pts(&buffer).ok().flatten(); + assert_eq!( + restored, + expected.map(gst::ClockTime::from_useconds), + "clip {clip} at {pts}us" + ); + } } }