From 155aad618be46a87085b1f50885db2256bdb1506 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 06:42:53 -0700 Subject: [PATCH 1/6] chore(quest): claim quest/m1/opus-preskip.md Quest-Claim: a9486673ff1cc482 From d338d34f2cea8b54d1f12d40fdd783580012e74c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 06:48:37 -0700 Subject: [PATCH 2/6] fix(moq-audio): keep a track's start when decoding from its first packet Decoding from frame 0 of group 0 drops only the codec's pre-skip; a mid-stream join still trims the catalog warmup. Co-Authored-By: Claude Opus 5.5 --- rs/moq-audio/src/decode/conformance.rs | 9 +-- rs/moq-audio/src/decode/consumer.rs | 89 +++++++++++++++++++-- rs/moq-audio/src/decode/test_data/README.md | 4 +- rs/moq-c/src/test.rs | 8 +- 4 files changed, 92 insertions(+), 18 deletions(-) diff --git a/rs/moq-audio/src/decode/conformance.rs b/rs/moq-audio/src/decode/conformance.rs index cf79e67c37b..405e908385e 100644 --- a/rs/moq-audio/src/decode/conformance.rs +++ b/rs/moq-audio/src/decode/conformance.rs @@ -239,12 +239,7 @@ async fn consumer_preserves_reference_samples_and_timeline() { pcm.extend(samples(&frame.data)); } } - let warmup = config.warmup.unwrap_or_default(); - let skip = (warmup.as_nanos() * vector.rate as u128 / 1_000_000_000) as usize * vector.channels as usize; - let reference = Vector { - reference: &vector.reference[skip * 4..], - ..*vector - }; - compare(&reference, "consumer", &pcm); + // Decoded from the track's first packet, so no warmup is trimmed. + compare(vector, "consumer", &pcm); } } diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 3e79ca5ee6e..30434504431 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -110,9 +110,12 @@ pub struct Consumer { remix: Option, options: Options, max_delay: std::time::Duration, - /// Prime after a join or loss, never after a declared endpoint. + /// Prime after a join or loss, never after a declared endpoint or at the track's + /// first packet, which the codec's own delay already primes. warmup: Duration, warmup_pending: bool, + /// Whether a packet was read or lost yet, so the track's first packet can be told apart. + started: bool, warmup_until: Option, resolved_sample_rate: u32, resolved_layout: Layout, @@ -224,6 +227,7 @@ impl Consumer { warmup, warmup_pending: !warmup.is_zero(), warmup_until: None, + started: false, resolved_sample_rate: sample_rate, resolved_layout: layout, next_start: None, @@ -295,17 +299,22 @@ impl Consumer { return Ok(Some(frame)); } - let mux_frame = match self.track.read().await? { + let (mux_frame, track_start) = match self.track.read().await? { Some(Encoded::Frame { group, frame }) => { + let first = !std::mem::replace(&mut self.started, true); // A later group after a declared end comes from a fresh encoder; the end's // own group carries the stopped encoder's final packets, trimmed below. if self.end.is_some() && group != self.end_group { self.restart()?; } - frame + // Nothing precedes group 0, and a group delivers from its frame 0 or not at + // all, so reading it first means decoding from the encoder's first packet. + // A finished group's end marker may come first; it carries no media. + (frame, first && group == 0) } // Behind whatever the resampler held from before the loss. Some(Encoded::Gap { timestamp, duration }) => { + self.started = true; self.restart()?; self.warmup_pending = !self.warmup.is_zero(); self.ready.push_back(Read::Gap { timestamp, duration }); @@ -360,11 +369,12 @@ impl Consumer { let epoch = *self.epoch.get_or_insert(mux_frame.timestamp); let delay = self.decoder.delay_remaining(); let decoded = self.decoder.decode(&mux_frame.payload)?; - // Anchored only once a packet decodes: a refused one primed nothing. - if self.warmup_pending { + // Anchored only once a packet decodes: a refused one primed nothing. The track's + // first packet has no history to warm up from: the codec's pre-skip, trimmed by + // the decoder, is all the priming its encoder declared. + if std::mem::take(&mut self.warmup_pending) && !track_start { let warmup = moq_net::Timestamp::try_from(self.warmup)?.convert(timestamp.scale())?; self.warmup_until = Some(timestamp.checked_add(warmup)?); - self.warmup_pending = false; } // Codec delay trimmed off the front is media this packet covered even // though no samples came out, so it still moves the packet after it along. @@ -796,6 +806,8 @@ mod tests { track, moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), ); + // Joined mid-stream: the track's first packet needs no warmup. + producer.seek(1).unwrap(); for epoch in 0..2 { if epoch != 0 && declared { producer.discontinuity().unwrap(); @@ -1031,6 +1043,8 @@ mod tests { track, moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), ); + // Joined mid-stream: the track's first packet needs no warmup. + producer.seek(1).unwrap(); producer .write(moq_mux::container::Frame { timestamp: Timestamp::from_micros(1_000_000).unwrap(), @@ -1064,6 +1078,69 @@ mod tests { ); } + /// Decodes a finished track of `packets` 20 ms Opus packets starting at group `group`, + /// keeping the encoder's declared warmup, and returns the first frame's stamp and the + /// frames decoded. + async fn opus_track_output(group: u64, packets: u64) -> (Option, usize) { + let mut encoder = Encoder::new(&EncodeConfig::new(48_000, Layout::Mono)).unwrap(); + let catalog = encoder.catalog(); + assert_eq!(catalog.warmup, Some(Duration::from_millis(80))); + + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + producer.seek(group).unwrap(); + let pcm = vec![0.25f32; encoder.frame_size()]; + for packet in 0..packets { + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_scale(packet * encoder.frame_size() as u64, 48_000).unwrap(), + duration: None, + payload: encoder.encode(&pcm).unwrap().payload, + keyframe: true, + }) + .unwrap(); + } + producer.finish().unwrap(); + + let mut consumer = Consumer::new( + &broadcast.consume(), + &catalog, + "audio", + Options { + max_delay: Duration::from_secs(10), + ..Options::new() + }, + ) + .await + .unwrap(); + let (mut first, mut frames) = (None, 0); + while let Some(frame) = next_frame(&mut consumer).await.unwrap() { + first.get_or_insert(frame.timestamp.as_micros()); + frames += frame.data.len() / size_of::(); + } + (first, frames) + } + + /// Decoding from the track's first packet drops only the codec's pre-skip: the encoder + /// started there, so there is no history to warm up from. A track shorter than the + /// warmup still decodes. + #[tokio::test(start_paused = true)] + async fn the_tracks_first_packet_keeps_all_but_pre_skip() { + assert_eq!(opus_track_output(0, 2).await, (Some(0), 2 * 960 - 312)); + } + + /// A join past the track's first group still drops its warmup after the pre-skip. + #[tokio::test(start_paused = true)] + async fn a_mid_stream_join_drops_its_warmup() { + assert_eq!(opus_track_output(1, 6).await, (Some(80_000), 6 * 960 - 312 - 3_840)); + } + /// A refused packet primed nothing, so the window starts at the first one that decodes. #[tokio::test(start_paused = true)] async fn warmup_starts_at_the_first_decoded_packet() { diff --git a/rs/moq-audio/src/decode/test_data/README.md b/rs/moq-audio/src/decode/test_data/README.md index d5f9c5f1fb6..cd2f39e6ec6 100644 --- a/rs/moq-audio/src/decode/test_data/README.md +++ b/rs/moq-audio/src/decode/test_data/README.md @@ -44,8 +44,8 @@ the same. This tests the supported ingest without expanding the container API. Reference PCM includes all decoded AAC padding and excludes the Opus `dOps` pre-skip. Backend tests remove exactly the backend's declared delay. Consumer -tests additionally remove the catalog's 80 ms Opus warmup from the reference; -AAC declares no warmup. No correlation search, gain fitting, channel permutation, +tests decode from each track's first packet, which trims no warmup, so they +compare against the whole reference. No correlation search, gain fitting, channel permutation, or per-run tolerance adjustment is allowed. The following maxima were measured on x86-64 Linux with the pinned Nix toolchain, diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index 54434728ffa..c9fb322c797 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -4812,12 +4812,14 @@ fn gray_rgba(width: u32, height: u32) -> Vec { /// A finished encoder reaches a C decoder as an end read in microseconds rather /// than vanishing: Opus pads a partial last frame, so the finish publishes where /// the source really stopped. The default start decodes the finished track from -/// its first group rather than waiting at its end. +/// its first group rather than waiting at its end, even when it is shorter than +/// the warmup a mid-stream join trims. #[test] fn audio_raw_decode_reads_the_encoder_end() { const START_US: u64 = 123_000; - // Past the 80 ms Opus warmup the decoder trims, ending on a partial frame. - const SAMPLES: usize = 4_700; + // One partial frame, shorter than the 80 ms warmup a mid-stream join trims: decoded + // from the track's first packet, only the codec's pre-skip is dropped. + const SAMPLES: usize = 860; let origin = id(moq_origin_create()); let path = b"audio-raw-end-test"; From d59ded613f233cadf80224ff5f9c5939f3cd3dc4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 06:48:47 -0700 Subject: [PATCH 3/6] quest: finish quest/m1/opus-preskip.md Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 - quest/m1/opus-preskip.md | 17 ----------------- 2 files changed, 18 deletions(-) delete mode 100644 quest/m1/opus-preskip.md diff --git a/quest/m1/README.md b/quest/m1/README.md index fe863117b99..7cc29e32f08 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -160,7 +160,6 @@ removed). - [Native rung disable](/quest/m1/simulcast-rung-disable.md) - OBS stops sending the top renditions its grant cannot fund, as the Rust/FFI publishers do, and a shaped uplink through moq-ffi proves the resume - [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the playout target is default in both languages; a manual browser proof and a native trace replay remain - [A/V clock](/quest/m1/av-clock.md) - the audio playhead drives Sync.reference while audio plays, through per-track sync handles -- [Decoding from a track's first packet keeps its start](/quest/m1/opus-preskip.md) - moq-audio drops only OpusHead's pre-skip from a track's first packet, keeping the 80 ms warmup for mid-stream joins - [Container reads drop backfill](/quest/m1/read-backfill.md) - the JS container consumer drops groups below its read cursor like Rust - [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - a buffered `` asks for the oldest group first, so a voice reply's opening arrives first, and audio's head hold shrinks to the floor - [The publish controls stay visible without a preview](/quest/m1/publish-controls.md) - ``'s controls no longer collapse with a live camera and no preview diff --git a/quest/m1/opus-preskip.md b/quest/m1/opus-preskip.md deleted file mode 100644 index 55fe1a2316b..00000000000 --- a/quest/m1/opus-preskip.md +++ /dev/null @@ -1,17 +0,0 @@ -# [S] Decoding from a track's first packet keeps its start - -## Goal - -moq-audio decoding an Opus track from its first packet outputs everything but -the codec's pre-skip, so recordings keep their first 80 ms and a track shorter -than that decodes to something, in every binding. - -## Plan - -Decided 2026-10-30 (maintainer, `/quest-plan`): from the track's first packet, -drop only the OpusHead pre-skip, as the codec defines. A decoder that joins -mid-stream keeps today's 80 ms warmup, since it has no history to warm up -from. Found in [moq-dev/moq#7033](https://github.com/moq-dev/moq/pull/7033). - -Test both cases: a short finished track decodes to its length minus -pre-skip, and a mid-stream join still drops its warmup. From dd0f2487844e360bdc082fd9428c419698f88fcd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 06:56:05 -0700 Subject: [PATCH 4/6] test(moq-audio): the roundtrip decodes its whole clip from the track start Co-Authored-By: Claude Opus 5.5 --- rs/moq-audio/tests/integration/roundtrip.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/rs/moq-audio/tests/integration/roundtrip.rs b/rs/moq-audio/tests/integration/roundtrip.rs index 86de7c061df..00d410c7b9a 100644 --- a/rs/moq-audio/tests/integration/roundtrip.rs +++ b/rs/moq-audio/tests/integration/roundtrip.rs @@ -95,12 +95,10 @@ async fn opus_roundtrip(case: Case) { .await .unwrap(); producer.finish().unwrap(); - let warmup = rendition.warmup.unwrap_or_default(); - let skipped = (warmup.as_nanos() * case.rate as u128 / 1_000_000_000) as usize; + // Decoded from the track's first packet, so only the codec's pre-skip is dropped. let mut pcm = Vec::new(); while let Some(frame) = next_frame(&mut consumer).await.unwrap() { - let expected = - 123_000 + warmup.as_micros() + pcm.len() as u128 / channels as u128 * 1_000_000 / case.rate as u128; + let expected = 123_000 + pcm.len() as u128 / channels as u128 * 1_000_000 / case.rate as u128; // Microsecond stamping rounds each chunk independently. assert!( frame.timestamp.as_micros().abs_diff(expected) <= 1, @@ -111,12 +109,12 @@ async fn opus_roundtrip(case: Case) { } assert_eq!( pcm.len(), - (expected_frames - skipped) * channels as usize, + expected_frames * channels as usize, "pre-skip, resampling and final padding preserve duration" ); for step in 0..2 { - let start = (step * expected_frames / 2).max(skipped) - skipped; - let end = (step + 1) * expected_frames / 2 - skipped; + let start = step * expected_frames / 2; + let end = (step + 1) * expected_frames / 2; let half = &pcm[start * channels as usize..end * channels as usize]; for channel in 0..channels as usize { let samples: Vec = half.iter().skip(channel).step_by(channels as usize).copied().collect(); From 8c32d84354ff06005f78233512062007a7fa5677 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 22:21:11 -0700 Subject: [PATCH 5/6] test(moq-audio): the burst join test starts mid-stream so it still tests a join Co-Authored-By: Claude Opus 5.5 --- rs/moq-audio/src/decode/consumer.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 51694de5d3b..e0029f76048 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -973,6 +973,8 @@ mod tests { track, moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), ); + // Joined mid-stream: the track's first packet needs no warmup. + producer.seek(1).unwrap(); // One 20 ms packet per group. let mut write = |packet: u64| { producer From 5d92b8d16566970b785b688f003946df2cd1dcdf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 08:43:20 -0800 Subject: [PATCH 6/6] test(moq-audio): set the track-start test's max delay through Delay main's decode::consumer::Options replaced max_delay with delay: Delay { min, max } in #7311, so the merged test no longer compiled. Co-Authored-By: Claude Opus 5.5 --- rs/moq-audio/src/decode/consumer.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index db7a041ccc2..9323e8eb52b 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -1222,7 +1222,10 @@ mod tests { &catalog, "audio", Options { - max_delay: Duration::from_secs(10), + delay: Delay { + max: Some(Duration::from_secs(10)), + ..Delay::default() + }, ..Options::new() }, )