From 76e58f901a4f980c3bb24b154a559867d59bbd54 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 20:06:45 -0700 Subject: [PATCH 01/22] chore: claim play resume quest Co-Authored-By: GPT-6 From f060e98d9cb8bbd27ffbf253e5e16453dc33657c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 22:55:06 -0700 Subject: [PATCH 02/22] fix(play): checkpoint sequential encoder segment playback Preserve decoded lifecycle events and buffered video/audio before replacing an encoder segment. Mocked quick-resume audio/video and shared-boundary tests passed; the original video regression lost half the pictures. Stable decoded segment metadata and full audio/play coverage are being validated. Unresolved: an independent declared end must not wait forever on a continuous peer. Combined A/V, gap/EOF/cancellation, resource bounds, required checks and cross-language interop remain. The quest is retained. Co-Authored-By: GPT-6 # Conflicts: # doc/setup/upgrade.md --- doc/setup/upgrade.md | 5 + rs/moq-audio/src/decode/consumer.rs | 144 ++++++++++++----- rs/moq-audio/src/decode/mod.rs | 4 +- rs/moq-audio/src/encode/producer.rs | 23 ++- rs/moq-audio/src/frame.rs | 2 +- rs/moq-audio/src/playback.rs | 3 +- rs/moq-audio/tests/import.rs | 12 +- rs/moq-audio/tests/roundtrip.rs | 20 ++- rs/moq-c/src/audio.rs | 3 +- rs/moq-cli/src/play/media.rs | 242 ++++++++++++++++++++-------- rs/moq-cli/src/play/timeline.rs | 104 ++++++++++++ rs/moq-cli/src/play/video.rs | 132 +++++++++++++-- rs/moq-ffi/src/audio.rs | 8 +- 13 files changed, 559 insertions(+), 143 deletions(-) diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index 1afdc054f35..cfddbea8b87 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -24,6 +24,11 @@ These land with the next breaking release, not the 2026-09-23 train. `catalog::hang::Consumer` with `.into()`, then read it through `catalog::Stream`. Its `Hang`, `Msf`, and `Closed` variants and `catalog::msf::Consumer::track` are private. +- **Rust audio reads carry track boundaries.** `moq_audio::decode::Consumer::read()` + returns `decode::Read` events. Handle `Frame { segment, frame }`, `Gap`, and + `End` separately; `None` still means the track is finished. An `End` can be + followed by final PCM in the same decoder segment before a fresh segment arrives. + Binding audio frame APIs keep their existing shape. - **JS catalog ranking names its media kind.** Replace `Catalog.ranked(renditions)` with `Catalog.Video.ranked(renditions)`. `Catalog.Audio.ranked(renditions)` orders audio by enabled state, bitrate, sample rate, channel count, and name. diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index d2b30e0eb1e..2e1694da67a 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -2,7 +2,7 @@ use std::collections::VecDeque; -use moq_mux::container::Read; +use moq_mux::container::Read as Encoded; use std::time::{Duration, Instant}; use bytes::Bytes; @@ -12,6 +12,32 @@ use crate::jitter::{self, Target}; use crate::resample::{Remix, Resampler}; use crate::{Activity, Error, Format, Frame, Layout}; +/// Decoded PCM and track boundaries in delivery order. +#[derive(Clone, Debug)] +pub enum Read { + /// PCM from one decoder segment, including its drained tail. + Frame { + /// The decoder segment; only equality matters. + segment: u64, + /// The decoded samples and their presentation time. + frame: Frame, + }, + /// Media was lost before the next frame. + Gap { + /// The furthest presentation point read before the loss. + timestamp: moq_net::Timestamp, + /// The upper bound on the missing media span. + duration: moq_net::Timestamp, + }, + /// The publisher stopped this encoder; final PCM may still follow in its segment. + End { + /// The decoder segment containing the stopped encoder's final samples. + segment: u64, + /// Samples at or past this point are padding and are trimmed. + end: moq_net::Timestamp, + }, +} + /// Where a consumer starts on a track that already holds groups. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[non_exhaustive] @@ -68,8 +94,7 @@ impl Options { /// /// The mirror of [`encode::Producer`](crate::encode::Producer): output format / /// sample rate / layout are fixed at construction, and -/// [`read`](Self::read) returns [`Frame`]s carrying the codec activity they -/// were decoded from. +/// [`read`](Self::read) returns [`Read`] events carrying decoded samples and track boundaries. pub struct Consumer { decoder: Decoder, track: moq_mux::container::Consumer, @@ -92,7 +117,9 @@ pub struct Consumer { next_start: Option, /// Frames decoded and not yet handed back, so a gap's tail can be returned /// ahead of the packet that exposed it. - ready: VecDeque, + ready: VecDeque, + /// Stable across all groups and the resampler tail of one decoder segment. + segment: u64, /// Codec activity spans the resampler's buffered output still covers. spans: VecDeque, /// Activity of the last span the output ran past, for the rounding samples the @@ -209,6 +236,7 @@ impl Consumer { resolved_layout: layout, next_start: None, ready: VecDeque::new(), + segment: 0, spans: VecDeque::new(), trailing: Activity::Active, epoch: None, @@ -258,7 +286,7 @@ impl Consumer { self.resolved_layout } - /// Read the next decoded PCM frame, or `None` when the track ends. + /// Read decoded PCM or a track boundary, or `None` when the track ends. /// /// [`Frame::activity`] reports whether the packet these samples came from /// coded audio. It describes where the frame begins, so a resampled @@ -270,34 +298,43 @@ impl Consumer { /// side stay anchored to their own packet timeline, so the hole is there to /// see. "Doesn't continue" allows for the quantization the stamps carry, which /// on a millisecond-stamped ingest is most of a millisecond. - pub async fn read(&mut self) -> Result, Error> { + pub async fn read(&mut self) -> Result, Error> { loop { if let Some(frame) = self.ready.pop_front() { return Ok(Some(frame)); } let mux_frame = match self.track.read().await? { - Some(Read::Frame { group, frame }) => { + Some(Encoded::Frame { group, frame }) => { // A later group after a declared end comes from a fresh encoder; the end's // own group carries the stopped encoder's final packets, trimmed below. if self.end.is_some() && group != self.end_group { self.restart()?; + self.segment += 1; } frame } - Some(Read::Gap { .. }) => { + Some(Encoded::Gap { timestamp, duration }) => { self.restart()?; + self.segment += 1; self.warmup_pending = !self.warmup.is_zero(); + self.ready.push_back(Read::Gap { timestamp, duration }); continue; } - Some(Read::End { group, end }) => { + Some(Encoded::End { group, end }) => { self.end = Some(end); self.end_group = group; self.frames_decoded = 0; self.terminal_start = None; - continue; + return Ok(Some(Read::End { + segment: self.segment, + end, + })); + } + None => { + let segment = self.segment; + return Ok(self.flush()?.map(|frame| Read::Frame { segment, frame })); } - None => return self.flush(), }; // Every packet the container hands over is an arrival, including one the @@ -325,7 +362,10 @@ impl Consumer { .is_some_and(|next| discontinuous(next, mux_frame.timestamp)) && let Some(frame) = self.gap()? { - self.ready.push_back(frame); + self.ready.push_back(Read::Frame { + segment: self.segment, + frame, + }); } let rate = self.decoder.sample_rate(); @@ -427,7 +467,10 @@ impl Consumer { // Queued rather than returned, so a tail drained at a gap earlier in this // same iteration still comes out first. The next turn of the loop pops it. if let Some(frame) = self.frame(pcm, timestamp, activity)? { - self.ready.push_back(frame); + self.ready.push_back(Read::Frame { + segment: self.segment, + frame, + }); } } } @@ -439,7 +482,10 @@ impl Consumer { /// as their own frame first rather than filtered together with what follows it. fn restart(&mut self) -> Result<(), Error> { if let Some(frame) = self.drain()? { - self.ready.push_back(frame); + self.ready.push_back(Read::Frame { + segment: self.segment, + frame, + }); } self.next_start = None; self.spans.clear(); @@ -648,6 +694,16 @@ fn rewind(timestamp: moq_net::Timestamp, frames: usize, sample_rate: u32) -> Res #[cfg(test)] mod tests { + async fn next_frame(consumer: &mut Consumer) -> Result, Error> { + loop { + match consumer.read().await? { + Some(Read::Frame { frame, .. }) => return Ok(Some(frame)), + Some(Read::Gap { .. } | Read::End { .. }) => {} + None => return Ok(None), + } + } + } + use moq_net::Timestamp; use super::*; @@ -696,7 +752,7 @@ mod tests { .unwrap(); let mut stamps = Vec::new(); let mut frames = 0; - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = next_frame(&mut consumer).await.unwrap() { stamps.push(frame.timestamp.as_micros()); frames += frame.data.len() / size_of::(); } @@ -765,7 +821,7 @@ mod tests { ) .await .unwrap(); - let frame = consumer.read().await.unwrap().unwrap(); + let frame = next_frame(&mut consumer).await.unwrap().unwrap(); assert!(frame.timestamp.as_micros() >= 1_080_000); let first = f32::from_le_bytes(frame.data[..4].try_into().unwrap()); assert!( @@ -814,9 +870,9 @@ mod tests { ) .await .unwrap(); - assert!(consumer.read().await.is_err()); + assert!(next_frame(&mut consumer).await.is_err()); let mut stamps = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = next_frame(&mut consumer).await.unwrap() { stamps.push(frame.timestamp.as_micros()); } assert_eq!(stamps, [100_000, 120_000]); @@ -866,7 +922,7 @@ mod tests { } producer.finish().unwrap(); let mut stamps = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = next_frame(&mut consumer).await.unwrap() { stamps.push(frame.timestamp.as_micros()); } assert_eq!(stamps.len(), 4, "each span's body and tail: {stamps:?}"); @@ -920,7 +976,7 @@ mod tests { } producer.write(&Frame::new(data.into(), Timestamp::ZERO)).unwrap(); - let frame = consumer.read().await.unwrap().expect("decoded frame"); + let frame = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); let samples = Format::F32.as_interleaved_f32(&frame.data, 2).unwrap(); assert_eq!(samples.len(), (960 - 312) * 2); for pair in samples.as_chunks::<2>().0.iter() { @@ -964,7 +1020,7 @@ mod tests { // 312 samples at 48 kHz is 6.5 ms. for (micros, frames) in [(0, 960 - 312), (13_500, 960), (33_500, 960)] { - let frame = consumer.read().await.unwrap().expect("decoded frame"); + let frame = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); assert_eq!(frame.timestamp.as_micros(), micros); assert_eq!(frame.data.len() / size_of::(), frames); } @@ -1020,7 +1076,7 @@ mod tests { .unwrap(); } - let first = consumer.read().await.unwrap().expect("decoded frame"); + let first = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); assert_eq!(first.timestamp.as_micros(), 0); // Continuity, not a fixed number: the second frame starts where the first @@ -1028,7 +1084,7 @@ mod tests { // exactly, because the resampler emits whole frames and its count per chunk // wobbles around the nominal ratio; a real hole (the samples it held back, or // the startup silence it dropped) is twenty times this tolerance. - let second = consumer.read().await.unwrap().expect("decoded frame"); + let second = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); let first_frames = (first.data.len() / size_of::()) as u128; let ends_at = first_frames * 1_000_000 / 48_000; let gap = second.timestamp.as_micros().abs_diff(ends_at); @@ -1080,10 +1136,10 @@ mod tests { .unwrap(); producer.finish().unwrap(); - let first = consumer.read().await.unwrap().expect("decoded frame"); + let first = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); let first_frames = first.data.len() / size_of::(); - let tail = consumer.read().await.unwrap().expect("flushed tail"); + let tail = next_frame(&mut consumer).await.unwrap().expect("flushed tail"); let tail_frames = tail.data.len() / size_of::(); // The 142 held-back frames at 44.1 kHz are ~155 at 48 kHz, plus the 69 the @@ -1101,7 +1157,7 @@ mod tests { // what the drain adds here is what the start dropped off the front. let total = first_frames + tail_frames; assert!((1105..=1120).contains(&total), "unexpected total: {total}"); - assert!(consumer.read().await.unwrap().is_none()); + assert!(next_frame(&mut consumer).await.unwrap().is_none()); } #[tokio::test] @@ -1163,7 +1219,7 @@ mod tests { let expected = first_dtx.expect("silence should enter Opus DTX"); let mut actual = None; - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = next_frame(&mut consumer).await.unwrap() { // 10 ms packets do not fill the 20 ms chunk, so the resampler hands back // nothing every other packet. Those must not surface as frames: a frame // with no samples reads as audio arriving, and carries an activity @@ -1237,7 +1293,7 @@ mod tests { producer.finish().unwrap(); let mut read = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = next_frame(&mut consumer).await.unwrap() { read.push((frame.timestamp.as_micros(), frame.data.len() / size_of::())); } read @@ -1387,7 +1443,7 @@ mod tests { write(&mut producer, 48_000, dtx.payload, false); producer.finish().unwrap(); - let frame = consumer.read().await.unwrap().expect("decoded frame"); + let frame = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); // The output begins with the first packet's samples, so it is stamped and // labelled from that packet. Rewinding from the terminal one instead drops it // most of a second into the future, carrying the DTX label with it. @@ -1492,17 +1548,20 @@ mod tests { // The pre-skip is trimmed off the first packet, so it decodes short. That // shortfall is codec delay, not a hole: without counting it the packet after // every stream start would read as a gap. - let first = consumer.read().await.unwrap().expect("decoded frame"); + let first = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); let frames = first.data.len() / size_of::(); assert!(frames < 960, "the pre-skip should be trimmed, got {frames} frames"); // The hole is real, so codec prediction starts over but stream-level pre-skip // does not. The audio is stamped where the packet says rather than 2.5 ms early. - let second = consumer.read().await.unwrap().expect("decoded frame"); + let second = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); assert_eq!(second.timestamp.as_micros(), 22_500); assert_eq!(second.data.len() / size_of::(), 960, "pre-skip was reapplied"); - let third = consumer.read().await.unwrap().expect("decoded frame after gap"); + let third = next_frame(&mut consumer) + .await + .unwrap() + .expect("decoded frame after gap"); let second_frames = second.data.len() / size_of::(); assert_eq!( third.timestamp, @@ -1569,7 +1628,10 @@ mod tests { .unwrap(); producer.cut(None).unwrap(); - consumer.read().await.unwrap().expect("one decoded frame per packet"); + next_frame(&mut consumer) + .await + .unwrap() + .expect("one decoded frame per packet"); series.push(crate::jitter::millis(consumer.delay())); } CLOCK.set(None); @@ -1640,7 +1702,7 @@ mod tests { .unwrap(); producer.cut(None).unwrap(); - let frame = consumer.read().await.unwrap().expect("decoded frame"); + let frame = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); assert!( frame.data.len() / size_of::() < encoder.frame_size(), "pre-skip is trimmed" @@ -1747,8 +1809,8 @@ mod tests { producer.cut(None).unwrap(); } - let first = consumer.read().await.unwrap().expect("first decoded frame"); - let second = consumer.read().await.unwrap().expect("second decoded frame"); + let first = next_frame(&mut consumer).await.unwrap().expect("first decoded frame"); + let second = next_frame(&mut consumer).await.unwrap().expect("second decoded frame"); let first_frames = first.data.len() / size_of::(); let expected = advance(first.timestamp, first_frames, 48_000).unwrap(); assert_eq!(second.timestamp, expected); @@ -1806,9 +1868,9 @@ mod tests { write(&mut producer, 2, encoder.encode(&pcm).unwrap().payload); producer.finish().unwrap(); - let first = consumer.read().await.unwrap().expect("first decoded frame"); - let _second = consumer.read().await.unwrap().expect("second decoded frame"); - let resumed = consumer.read().await.unwrap().expect("resumed decoded frame"); + let first = next_frame(&mut consumer).await.unwrap().expect("first decoded frame"); + let _second = next_frame(&mut consumer).await.unwrap().expect("second decoded frame"); + let resumed = next_frame(&mut consumer).await.unwrap().expect("resumed decoded frame"); let first_frames = first.data.len() / size_of::(); let resumed_frames = resumed.data.len() / size_of::(); assert!(first_frames < frame_size, "the first epoch trims pre-skip"); @@ -1859,7 +1921,7 @@ mod tests { }) .unwrap(); - let frame = consumer.read().await.unwrap().expect("decoded frame"); + let frame = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); assert_eq!( Format::F32.as_interleaved_f32(&frame.data, 1).unwrap().as_ref(), samples @@ -1916,7 +1978,7 @@ mod tests { // The whole packet decodes: one 20 ms Opus frame at 48 kHz, less the pre-skip // trimmed off the first packet. Reading the fragment as legacy hands the codec // a slice of the moof instead, which still decodes, just to a shorter buffer. - let frame = consumer.read().await.unwrap().expect("decoded frame"); + let frame = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); // `as_micros`, not `==`: the CMAF path carries the fmp4 timescale and // `Timestamp`'s equality is structural, so the scales would have to match too. assert_eq!(frame.timestamp.as_micros(), 0); diff --git a/rs/moq-audio/src/decode/mod.rs b/rs/moq-audio/src/decode/mod.rs index 6f4500054c4..dc0850838ff 100644 --- a/rs/moq-audio/src/decode/mod.rs +++ b/rs/moq-audio/src/decode/mod.rs @@ -5,7 +5,7 @@ //! //! Entry points, high to low level: //! - [`Consumer`] subscribes to a track and hands back decoded -//! [`Frame`](crate::Frame)s. +//! [`Read`] events. //! - [`Decoder`] decodes packets you supply (bring your own payloads) into //! [`Decoded`] interleaved `f32` samples. //! @@ -19,6 +19,6 @@ mod consumer; mod decoded; mod decoder; -pub use consumer::{Consumer, Options, Output, Start}; +pub use consumer::{Consumer, Options, Output, Read, Start}; pub use decoded::Decoded; pub use decoder::{Config, Decoder, Kind}; diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index 85465c01af4..ffc9322de68 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -559,6 +559,16 @@ impl Producer { #[cfg(test)] mod tests { + async fn decoded_frame(consumer: &mut crate::decode::Consumer) -> Result, Error> { + loop { + match consumer.read().await? { + Some(crate::decode::Read::Frame { frame, .. }) => return Ok(Some(frame)), + Some(crate::decode::Read::Gap { .. } | crate::decode::Read::End { .. }) => {} + None => return Ok(None), + } + } + } + use std::time::Duration; use super::*; @@ -661,7 +671,7 @@ mod tests { producer.finish().unwrap(); let mut decoded = Vec::new(); - while let Some(frame) = audio.read().await.unwrap() { + while let Some(frame) = decoded_frame(&mut audio).await.unwrap() { let pcm = Format::F32.as_interleaved_f32(&frame.data, 1).unwrap(); decoded.extend_from_slice(&pcm); } @@ -834,7 +844,10 @@ mod tests { .unwrap(); producer.write(&full_frame(0)).unwrap(); - let first = audio.read().await.unwrap().expect("first packet before the reanchor"); + let first = decoded_frame(&mut audio) + .await + .unwrap() + .expect("first packet before the reanchor"); assert_eq!(first.data.len() / size_of::(), 960 - 312); producer.reanchor(); @@ -842,7 +855,7 @@ mod tests { producer.finish().unwrap(); let mut resumed_frames = 0; - while let Some(frame) = audio.read().await.unwrap() { + while let Some(frame) = decoded_frame(&mut audio).await.unwrap() { if frame.timestamp.as_micros() >= 1_000_000 { resumed_frames += frame.data.len() / size_of::(); } @@ -883,7 +896,7 @@ mod tests { let mut entered_dtx = false; for index in 0..100 { producer.write(&pcm_frame(&silence, index * 20_000)).unwrap(); - let consumed = consumer.read().await.unwrap().expect("one decoded frame"); + let consumed = decoded_frame(&mut consumer).await.unwrap().expect("one decoded frame"); assert_eq!(producer.activity(), consumed.activity); if consumed.activity.is_dtx() { entered_dtx = true; @@ -899,7 +912,7 @@ mod tests { }) .collect(); producer.write(&pcm_frame(&active, 2_000_000)).unwrap(); - let consumed = consumer.read().await.unwrap().expect("one decoded frame"); + let consumed = decoded_frame(&mut consumer).await.unwrap().expect("one decoded frame"); assert_eq!(producer.activity(), Activity::Active); assert_eq!(consumed.activity, Activity::Active); } diff --git a/rs/moq-audio/src/frame.rs b/rs/moq-audio/src/frame.rs index 2547b4a9b25..200641a97ee 100644 --- a/rs/moq-audio/src/frame.rs +++ b/rs/moq-audio/src/frame.rs @@ -5,7 +5,7 @@ use crate::Activity; /// One unit of raw PCM crossing the codec boundary: what /// [`encode::Producer::write`](crate::encode::Producer::write) takes and what -/// [`decode::Consumer::read`](crate::decode::Consumer::read) returns. +/// [`decode::Read::Frame`](crate::decode::Read::Frame) carries. /// /// Just a payload, a presentation timestamp, and whether the packet these /// samples came from coded any audio. PCM layout (format / sample rate / channel count) diff --git a/rs/moq-audio/src/playback.rs b/rs/moq-audio/src/playback.rs index 798bb5984dc..9277cf455ed 100644 --- a/rs/moq-audio/src/playback.rs +++ b/rs/moq-audio/src/playback.rs @@ -17,7 +17,8 @@ //! //! let mut sink = engine.sink(input)?; //! -//! while let Some(frame) = audio.read().await? { +//! while let Some(event) = audio.read().await? { +//! let moq_audio::decode::Read::Frame { frame, .. } = event else { continue }; //! let write = sink.write(&frame.data)?; //! if write.dropped_sample_frames > 0 { //! eprintln!("dropped {} live audio frames", write.dropped_sample_frames); diff --git a/rs/moq-audio/tests/import.rs b/rs/moq-audio/tests/import.rs index a1a5a147e43..0e1faa69cb7 100644 --- a/rs/moq-audio/tests/import.rs +++ b/rs/moq-audio/tests/import.rs @@ -34,7 +34,7 @@ async fn ts_surround_opus_decodes() { // Canonical 5.1 order: FL, FR, FC, LFE, SL, SR. let mut energy = [0.0f64; 6]; let mut frames = 0; - while let Some(frame) = tokio::time::timeout(Duration::from_secs(5), consumer.read()) + while let Some(frame) = tokio::time::timeout(Duration::from_secs(5), next_frame(&mut consumer)) .await .expect("decoded frame timed out") .unwrap() @@ -62,3 +62,13 @@ async fn ts_surround_opus_decodes() { ); } } + +async fn next_frame(consumer: &mut decode::Consumer) -> Result, moq_audio::Error> { + loop { + match consumer.read().await? { + Some(decode::Read::Frame { frame, .. }) => return Ok(Some(frame)), + Some(decode::Read::Gap { .. } | decode::Read::End { .. }) => {} + None => return Ok(None), + } + } +} diff --git a/rs/moq-audio/tests/roundtrip.rs b/rs/moq-audio/tests/roundtrip.rs index 34ba2750638..68c64c820e3 100644 --- a/rs/moq-audio/tests/roundtrip.rs +++ b/rs/moq-audio/tests/roundtrip.rs @@ -70,7 +70,7 @@ async fn opus_round_trip_48k_stereo() { let mut total_frames = 0u64; let mut total_energy = 0.0f64; - while let Some(frame) = tokio::time::timeout(Duration::from_secs(5), consumer.read()) + while let Some(frame) = tokio::time::timeout(Duration::from_secs(5), next_frame(&mut consumer)) .await .expect("decoded frame timed out") .unwrap() @@ -146,7 +146,7 @@ async fn opus_round_trip_44100_s16_resampled() { producer.finish().unwrap(); let mut total_bytes = 0u64; - while let Some(frame) = tokio::time::timeout(Duration::from_secs(5), consumer.read()) + while let Some(frame) = tokio::time::timeout(Duration::from_secs(5), next_frame(&mut consumer)) .await .expect("decoded frame timed out") .unwrap() @@ -191,7 +191,7 @@ async fn pcm_round_trip_is_lossless() { .unwrap(); producer.finish().unwrap(); - let frame = consumer.read().await.unwrap().unwrap(); + let frame = next_frame(&mut consumer).await.unwrap().unwrap(); assert_eq!(frame.timestamp, Timestamp::from_micros(123_000).unwrap()); let decoded: Vec = frame .data @@ -201,7 +201,7 @@ async fn pcm_round_trip_is_lossless() { .map(|sample| f32::from_le_bytes([sample[0], sample[1], sample[2], sample[3]])) .collect(); assert_eq!(decoded, samples); - assert!(consumer.read().await.unwrap().is_none()); + assert!(next_frame(&mut consumer).await.unwrap().is_none()); } /// A 5.1 broadcast carries only its channel count in the catalog, which reads @@ -257,7 +257,7 @@ async fn pcm_five_one_reads_back_at_any_layout() { for (mut consumer, (_, layout, frame)) in consumers.into_iter().zip(cases) { assert_eq!(consumer.layout(), layout); - let decoded = consumer.read().await.unwrap().unwrap(); + let decoded = next_frame(&mut consumer).await.unwrap().unwrap(); let samples: Vec = decoded .data .as_chunks::<4>() @@ -271,3 +271,13 @@ async fn pcm_five_one_reads_back_at_any_layout() { } } } + +async fn next_frame(consumer: &mut decode::Consumer) -> Result, moq_audio::Error> { + loop { + match consumer.read().await? { + Some(decode::Read::Frame { frame, .. }) => return Ok(Some(frame)), + Some(decode::Read::Gap { .. } | decode::Read::End { .. }) => {} + None => return Ok(None), + } + } +} diff --git a/rs/moq-c/src/audio.rs b/rs/moq-c/src/audio.rs index be2ef17ba0c..a7260f01770 100644 --- a/rs/moq-c/src/audio.rs +++ b/rs/moq-c/src/audio.rs @@ -259,7 +259,8 @@ impl Audio { biased; _ = &mut close => return Ok(()), frame = consumer.read() => match frame { - Ok(Some(frame)) => frame, + Ok(Some(moq_audio::decode::Read::Frame { frame, .. })) => frame, + Ok(Some(moq_audio::decode::Read::Gap { .. } | moq_audio::decode::Read::End { .. })) => continue, Ok(None) => return Ok(()), // One packet the codec rejected is that packet's problem: the // decoder stays usable, so drop it and keep the subscription rather diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 555eb84165a..16c3c477d0f 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -16,7 +16,7 @@ use tokio::time::Instant; use super::args::Args; use super::output::{Output, Sink, Speaker}; use super::playback::{Kind, Playback, joined}; -use super::timeline::{AudioTimeline, Presentation, fit, timestamp}; +use super::timeline::{AudioTimeline, Presentation, Role, fit, timestamp}; use super::video::Video; use super::window::Event; @@ -409,6 +409,10 @@ async fn play_audio( // comes and lets `fit` hold the sink on the target instead. The window // schedules video against where the speaker actually is, which keeps the two // together whatever the target does. + let mut role = Role::new(presentation.clone(), changed.clone(), Kind::Audio); + let received = Mutex::new(VecDeque::new()); + let arrived = tokio::sync::Notify::new(); + let max_delay = consumer.max_delay(); let sample_rate = consumer.sample_rate(); let layout = consumer.layout(); let channels = layout.channels(); @@ -435,7 +439,7 @@ async fn play_audio( // decoder's latency budget says: anything longer is what that budget chose to // skip, so playing it as silence would hand back the delay the skip avoided. // Past it the sink skips the hole and the clock re-anchors, as it does today. - let fill_max = samples(consumer.max_delay()); + let fill_max = samples(max_delay); let mut timeline = AudioTimeline::default(); @@ -443,80 +447,124 @@ async fn play_audio( // all logs once rather than once per packet. let mut dropping = false; - loop { - let frame = match consumer.read().await { - Ok(Some(frame)) => frame, - Ok(None) => break, - // One bad packet is that packet's problem: the decoder stays usable, so - // skip it rather than ending playback and taking the video window down - // with it. - Err(err @ moq_audio::Error::Decode(_)) => { - if dropping { - tracing::debug!(%err, "dropping an audio frame"); - } else { - tracing::warn!(%err, "dropping an audio frame"); - dropping = true; - } - continue; + let receive = async { + loop { + let event = match consumer.read().await { + Err(err) if !matches!(err, moq_audio::Error::Decode(_)) => return Err(err.into()), + event => event, + }; + let ended = matches!(event, Ok(None)); + let delay = consumer.delay(); + received.lock().unwrap().push_back((event, delay)); + arrived.notify_one(); + if ended { + return Ok::<_, anyhow::Error>(()); } - Err(err) => return Err(err.into()), - }; - dropping = false; - - let length = frame.data.len() / stride; - let start = timestamp(frame.timestamp); - let timing = timeline.push(start, length, sample_rate, fill_max); - - // A rewind or a hole too large to fill starts a new playback sink. The old - // sink has no media clock, so its buffered audio cannot be carried across a - // timeline region the player skipped. - if timing.reset_sink { - drop(sink); - presentation.lock().unwrap().restarted(); - sink = speaker.sink(input.clone())?; - dry = true; } + }; + let play = async { + let mut end_segment = None; + loop { + let (event, delay) = loop { + if let Some(event) = received.lock().unwrap().pop_front() { + break event; + } + arrived.notified().await; + }; + let frame = match event { + Ok(Some(moq_audio::decode::Read::Frame { segment, frame })) => { + if end_segment.take_if(|end| *end != segment).is_some() { + // A declared boundary preserves the old sink, unlike an undeclared hole. + drain(sink).await; + role.resume().await; + timeline = AudioTimeline::default(); + sink = speaker.sink(input.clone())?; + dry = true; + } + frame + } + Ok(Some(moq_audio::decode::Read::End { segment, .. })) => { + end_segment = Some(segment); + continue; + } + Ok(Some(moq_audio::decode::Read::Gap { .. })) => { + end_segment = None; + continue; + } + Ok(None) => break, + // One bad packet is that packet's problem: the decoder stays usable, so + // skip it rather than ending playback and taking the video window down + // with it. + Err(err @ moq_audio::Error::Decode(_)) => { + if dropping { + tracing::debug!(%err, "dropping an audio frame"); + } else { + tracing::warn!(%err, "dropping an audio frame"); + dropping = true; + } + continue; + } + Err(err) => return Err(err.into()), + }; + dropping = false; + + let length = frame.data.len() / stride; + let start = timestamp(frame.timestamp); + let timing = timeline.push(start, length, sample_rate, fill_max); + + // A rewind or a hole too large to fill starts a new playback sink. The old + // sink has no media clock, so its buffered audio cannot be carried across a + // timeline region the player skipped. + if timing.reset_sink { + drop(sink); + presentation.lock().unwrap().restarted(); + sink = speaker.sink(input.clone())?; + dry = true; + } - // A hole in the media is a hole in the audio, not a splice. Handing the next - // frame straight to the speaker shortens the track by the missing duration, - // which leaves it running ahead of media time until the clock below - // re-anchors, taking the video with it. So the hole goes in as silence ahead - // of the frame, and the pair is fitted to the target as one write. - let buffered = samples(sink.buffered()); - dry |= buffered == 0; - // Capped at the age budget: audio older than it is skipped rather than held, - // so a deeper target could never fill. The advertised floor needs the cap, - // being a number the publisher declared about itself, unbounded. The budget - // is also what the sink's ring was sized to hold. - let target = samples(consumer.delay().min(consumer.max_delay()).max(AUDIO_BUFFER_MIN)); - let fit = fit(dry, buffered, target, slack, timing.silence + length as u64); - dry = false; - - // Playback drops stay on the live timeline; retrying them would add latency, - // and the sink already reports them in its logs. - let mut quiet = fit.pad + timing.silence.saturating_sub(fit.skip); - while quiet > 0 { - let part = (quiet as usize * stride).min(silence.len()); - sink.write(&silence[..part])?; - quiet -= (part / stride) as u64; - } - let skip = fit.skip.saturating_sub(timing.silence) as usize * stride; - sink.write(&frame.data[skip.min(frame.data.len())..])?; - - // Anchor the playout clock on where the speaker has actually reached, which - // is the only half of the pipeline that cannot skip ahead. A move has to - // wake the window: it is asleep on a deadline computed from the old anchor, - // and every queued frame is due earlier now. - let moved = presentation - .lock() - .unwrap() - .audio(timing.end, sink.buffered(), Instant::now().into_std()); - if moved { - changed.notify_one(); - output.send(Event::Wake); + // A hole in the media is a hole in the audio, not a splice. Handing the next + // frame straight to the speaker shortens the track by the missing duration, + // which leaves it running ahead of media time until the clock below + // re-anchors, taking the video with it. So the hole goes in as silence ahead + // of the frame, and the pair is fitted to the target as one write. + let buffered = samples(sink.buffered()); + dry |= buffered == 0; + // Capped at the age budget: audio older than it is skipped rather than held, + // so a deeper target could never fill. The advertised floor needs the cap, + // being a number the publisher declared about itself, unbounded. The budget + // is also what the sink's ring was sized to hold. + let target = samples(delay.min(max_delay).max(AUDIO_BUFFER_MIN)); + let fit = fit(dry, buffered, target, slack, timing.silence + length as u64); + dry = false; + + // Playback drops stay on the live timeline; retrying them would add latency, + // and the sink already reports them in its logs. + let mut quiet = fit.pad + timing.silence.saturating_sub(fit.skip); + while quiet > 0 { + let part = (quiet as usize * stride).min(silence.len()); + sink.write(&silence[..part])?; + quiet -= (part / stride) as u64; + } + let skip = fit.skip.saturating_sub(timing.silence) as usize * stride; + sink.write(&frame.data[skip.min(frame.data.len())..])?; + + // Anchor the playout clock on where the speaker has actually reached, which + // is the only half of the pipeline that cannot skip ahead. A move has to + // wake the window: it is asleep on a deadline computed from the old anchor, + // and every queued frame is due earlier now. + let moved = presentation + .lock() + .unwrap() + .audio(timing.end, sink.buffered(), Instant::now().into_std()); + if moved { + changed.notify_one(); + output.send(Event::Wake); + } } - } + Ok::<_, anyhow::Error>(sink) + }; + let (_, sink) = tokio::try_join!(receive, play)?; Ok(sink) } @@ -543,6 +591,58 @@ mod tests { const PACKET: u64 = 960; const PACKET_DURATION: Duration = Duration::from_millis(20); + #[tokio::test] + async fn a_quick_audio_resume_preserves_the_old_speaker_tail() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Audio)); + let mut options = moq_audio::decode::Options::new(); + options.delay = Some(delay); + options.max_delay = delay; + options.output.format = moq_audio::Format::F32; + let consumer = moq_audio::decode::Consumer::new( + &broadcast.consume(), + &AudioConfig::new(AudioCodec::Pcm, SAMPLE_RATE, 1), + "audio", + options, + ) + .await + .unwrap(); + let recorder = Recorder::default(); + let playback = AudioPlayback { + changed: Default::default(), + speaker: recorder.clone(), + presentation: Arc::new(Mutex::new(Presentation::new(delay))), + latency: delay, + output: recorder.clone(), + }; + let task = tokio::spawn(async move { + play_audio(consumer, playback).await.unwrap().finish().await; + }); + producer.write(packet(0, 0.25)).unwrap(); + tokio::task::yield_now().await; + producer.discontinuity().unwrap(); + producer.write(packet(0, 0.5)).unwrap(); + producer.finish().unwrap(); + task.await.unwrap(); + let played = recorder.played(); + let old: Vec<_> = played.iter().filter(|part| part.sample == 0.25).collect(); + let new: Vec<_> = played.iter().filter(|part| part.sample == 0.5).collect(); + assert_eq!( + old.iter().map(|part| part.to - part.from).sum::(), + PACKET_DURATION + ); + assert_eq!( + new.iter().map(|part| part.to - part.from).sum::(), + PACKET_DURATION + ); + assert!(old.last().unwrap().to <= new.first().unwrap().from); + } + /// A mono PCM rendition, published and named in the catalog until dropped. fn rendition( broadcast: &moq_net::broadcast::Producer, diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index 53f5e41ece0..c3ceb149c07 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -2,6 +2,7 @@ use std::time::{Duration, Instant}; +use super::playback::Kind; use hang::moq_net::Timestamp; /// The playout clock: one anchor the window and the speaker both present @@ -28,6 +29,9 @@ pub(super) struct Presentation { /// audio frame pins the anchor outright instead of pacing against media it /// will never play. restart: bool, + /// Segment reached by each active playback role. + roles: [Option; 2], + segment: u64, } impl Presentation { @@ -37,9 +41,34 @@ impl Presentation { delay, speaker: false, restart: false, + roles: [None; 2], + segment: 0, } } + pub(super) fn started(&mut self, role: Kind) { + self.roles[role as usize] = Some(0); + } + + pub(super) fn finished(&mut self, role: Kind) { + self.roles[role as usize] = None; + } + + /// Rebase once every active role has played out the previous segment. + pub(super) fn resume(&mut self, role: Kind, segment: u64) -> bool { + self.roles[role as usize] = Some(segment); + if self.roles.iter().flatten().any(|ready| *ready < segment) { + return false; + } + if self.segment < segment { + self.segment = segment; + self.pacer = moq_mux::Pacer::default().with_delay(self.delay); + self.speaker = false; + self.restart = false; + } + true + } + /// Fold an arriving video frame into the anchor, unless the speaker owns it, /// reporting whether that moved the schedule. /// @@ -123,6 +152,54 @@ impl Presentation { } } +/// Releases a role when its track ends or the task is cancelled. +pub(super) struct Role { + presentation: std::sync::Arc>, + changed: std::sync::Arc, + role: Kind, + segment: u64, +} + +impl Role { + pub(super) fn new( + presentation: std::sync::Arc>, + changed: std::sync::Arc, + role: Kind, + ) -> Self { + presentation.lock().unwrap().started(role); + Self { + presentation, + changed, + role, + segment: 0, + } + } + + pub(super) async fn resume(&mut self) { + self.segment += 1; + self.presentation.lock().unwrap().resume(self.role, self.segment); + self.changed.notify_waiters(); + loop { + let changed = self.changed.notified(); + // Register before inspecting shared state so a peer cannot finish between them. + tokio::pin!(changed); + changed.as_mut().enable(); + if self.presentation.lock().unwrap().resume(self.role, self.segment) { + self.changed.notify_waiters(); + return; + } + changed.await; + } + } +} + +impl Drop for Role { + fn drop(&mut self) { + self.presentation.lock().unwrap().finished(self.role); + self.changed.notify_waiters(); + } +} + /// A wire timestamp as a duration from the start of the track. pub(super) fn timestamp(timestamp: Timestamp) -> Duration { Duration::from_micros(timestamp.as_micros().min(u64::MAX as u128) as u64) @@ -238,6 +315,33 @@ mod tests { Timestamp::from_millis(millis).unwrap() } + #[test] + fn a_segment_rebases_only_after_both_playback_roles_drain() { + let start = Instant::now(); + let mut presentation = Presentation::new(DELAY); + presentation.started(Kind::Video); + presentation.started(Kind::Audio); + presentation.audio(Duration::from_secs(10), DELAY, start); + let old = presentation.due(ms(10_000)); + assert!(!presentation.resume(Kind::Video, 1)); + assert_eq!(presentation.due(ms(10_000)), old); + assert!(presentation.resume(Kind::Audio, 1)); + assert_eq!(presentation.due(ms(10_000)), None); + presentation.audio(Duration::from_secs(1), DELAY, start); + assert!(presentation.resume(Kind::Video, 1)); + assert_eq!( + presentation.due(ms(1_000)), + Some(start + DELAY), + "one rebase per segment" + ); + assert!(!presentation.resume(Kind::Audio, 2)); + presentation.finished(Kind::Video); + assert!( + presentation.resume(Kind::Audio, 2), + "a closed track cannot hold its peer forever" + ); + } + /// Earlier arrivals pull the anchor forward so playback converges on the live /// edge without giving up the configured delay. #[test] diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 6bc0487bcba..4e6f83d1343 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -9,7 +9,7 @@ use tokio::time::Instant; use super::buffer::Buffer; use super::output::Output; -use super::timeline::Presentation; +use super::timeline::{Presentation, Role}; use super::window::Event; /// A few surfaces, independent of the requested playout delay. A codec batch @@ -26,6 +26,7 @@ enum Step { Decode, /// The track ended: drain what the codec still holds. Flush, + Resume, } /// When the window's queue has room for another picture, or `None` if it does @@ -71,22 +72,30 @@ impl Decoder for moq_video::decode::Sink { impl Video { pub async fn run(self, mut track: Track, mut decoder: impl Decoder) -> anyhow::Result<()> { - let buffer = Mutex::new(Buffer::default()); + let closed = std::sync::atomic::AtomicBool::new(false); + let buffers = Mutex::new(VecDeque::from([Buffer::default()])); + let mut role = Role::new( + self.presentation.clone(), + self.changed.clone(), + super::playback::Kind::Video, + ); let receive = async { // The group of a declared end whose restart waits for a later group, so the // media buffered before the end still plays out. let mut end = None; + loop { let read = track.read().await?; - let mut buffer = buffer.lock().unwrap(); - let before = buffer.generation; + let mut buffers = buffers.lock().unwrap(); + let active = buffers.len() == 1; + let before = buffers.front().unwrap().generation; let ended = match read { Some(Read::Frame { group, frame }) => { if end.take_if(|end| *end != group).is_some() { - buffer.clear(); - self.presentation.lock().unwrap().video_restarted(); + buffers.push_back(Buffer::default()); } - if self + if active + && buffers.len() == 1 && self .presentation .lock() .unwrap() @@ -94,25 +103,31 @@ impl Video { { self.output.send(Event::Wake); } - buffer.push(frame, self.max_delay); + buffers.back_mut().unwrap().push(frame, self.max_delay); false } // Applied in delivery order, so the restart lands before the frame after it. Some(Read::Gap { .. }) => { end = None; - buffer.clear(); + buffers.clear(); + buffers.push_back(Buffer::default()); + buffers.front_mut().unwrap().clear(); + buffers.front_mut().unwrap().generation = before + 1; self.presentation.lock().unwrap().video_restarted(); false } Some(Read::End { group, .. }) => { end = Some(group); + buffers.back_mut().unwrap().ended = true; false } None => { - buffer.ended = true; + closed.store(true, std::sync::atomic::Ordering::Relaxed); + buffers.back_mut().unwrap().ended = true; true } }; + let buffer = buffers.front().unwrap(); if buffer.generation != before { self.frames.lock().unwrap().clear(); self.output.send(Event::Wake); @@ -138,7 +153,8 @@ impl Video { let mut pending = VecDeque::new(); loop { let (current, oldest, ended) = { - let buffer = buffer.lock().unwrap(); + let buffers = buffers.lock().unwrap(); + let buffer = buffers.front().unwrap(); (buffer.generation, buffer.oldest(), buffer.ended) }; if generation.is_some_and(|generation| generation != current) { @@ -185,7 +201,19 @@ impl Video { } else if generation.is_some() { Step::Flush } else { - break; + // A new encoder waits for every old picture to be presented. + if closed.load(std::sync::atomic::Ordering::Relaxed) && buffers.lock().unwrap().len() == 1 { + break; + } else if !queue.is_empty() { + Step::Wait( + queue + .back() + .and_then(|frame| presentation.due(frame.timestamp)) + .filter(|at| *at > now), + ) + } else { + Step::Resume + } } }; let frames = match step { @@ -201,9 +229,11 @@ impl Video { continue; } Step::Decode => { - let frame = buffer + let frame = buffers .lock() .unwrap() + .front_mut() + .unwrap() .pop() .expect("no await since inspecting the buffer"); generation = Some(current); @@ -212,6 +242,32 @@ impl Video { // joined receiver continues observing arrivals during this await. decoder.decode(frame).await? } + Step::Resume => { + let next = { + let mut buffers = buffers.lock().unwrap(); + if buffers.len() > 1 { + buffers.pop_front(); + true + } else { + false + } + }; + if !next { + if closed.load(std::sync::atomic::Ordering::Relaxed) { + break; + } + self.changed.notified().await; + continue; + } + role.resume().await; + let buffers = buffers.lock().unwrap(); + let mut presentation = self.presentation.lock().unwrap(); + for frame in &buffers.front().unwrap().frames { + presentation.video(frame.timestamp, Instant::now().into_std()); + } + self.output.send(Event::Wake); + continue; + } Step::Flush => { generation = None; held.clear(); @@ -225,7 +281,7 @@ impl Video { } // A codec may return pictures it held across the keyframe that // resumed a skipped timeline. Those pictures no longer have a slot. - let floor = buffer.lock().unwrap().floor; + let floor = buffers.lock().unwrap().front().unwrap().floor; pending.extend( frames .into_iter() @@ -540,6 +596,54 @@ mod tests { task.await.unwrap().unwrap(); } + #[tokio::test] + async fn a_quick_resume_flushes_and_plays_every_old_picture_first() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let media = media(delay); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = track + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(delay)) + .await + .unwrap(); + let format = moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + ); + for ms in [0, 33, 66] { + producer.write(frame(ms, ms == 0)).unwrap(); + } + producer.discontinuity().unwrap(); + for ms in [0, 33, 66] { + producer.write(frame(ms, ms == 0)).unwrap(); + } + producer.finish().unwrap(); + let decoder = Buffered::new(2); + let flushed = decoder.flushed.clone(); + let task = + tokio::spawn(playback(&media, delay).run(moq_mux::container::Consumer::new(subscriber, format), decoder)); + let mut shown = Vec::new(); + for _ in 0..1_000 { + tokio::task::yield_now().await; + if let Some(timestamp) = media.output.present(&media) { + shown.push(timestamp.as_millis()); + } + tokio::time::advance(Duration::from_millis(1)).await; + } + task.await.unwrap().unwrap(); + assert_eq!(shown, [0, 33, 66, 0, 33, 66]); + assert_eq!( + *flushed.lock().unwrap(), + 2, + "each encoder drains before its replacement" + ); + } + /// Play one burst of 33 ms pictures, given by slot in decode order, through a /// codec holding `depth` pictures, and check the window shows every one on /// time. diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index 15f993f7ae7..faf8ea07742 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -342,7 +342,13 @@ struct ConsumerInner { impl ConsumerInner { async fn next(&mut self) -> Result, MoqError> { - Ok(self.consumer.read().await?.map(Into::into)) + loop { + match self.consumer.read().await? { + Some(moq_audio::decode::Read::Frame { frame, .. }) => return Ok(Some(frame.into())), + Some(moq_audio::decode::Read::Gap { .. } | moq_audio::decode::Read::End { .. }) => {} + None => return Ok(None), + } + } } } From ce05a37338f268cba7cedd5e15b229de66d34787 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 20:29:39 -0700 Subject: [PATCH 03/22] test(play): distinguish loss from declared encoder ends Run the existing discard oracle against a real skipped group, while declared ends drain old pictures. All273 audio/player tests pass. The independent-role end policy remains unresolved and the quest stays open. Co-Authored-By: GPT-6 --- rs/moq-cli/src/play/timeline.rs | 4 +++- rs/moq-cli/src/play/video.rs | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index c3ceb149c07..ba8644bdf12 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -195,7 +195,9 @@ impl Role { impl Drop for Role { fn drop(&mut self) { - self.presentation.lock().unwrap().finished(self.role); + if let Ok(mut presentation) = self.presentation.lock() { + presentation.finished(self.role); + } self.changed.notify_waiters(); } } diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 4e6f83d1343..b10f547009e 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -498,7 +498,7 @@ mod tests { assert_eq!(shown, [0, 33, 66, 99], "reordered output was not presented in order"); } #[tokio::test] - async fn a_discontinuity_discards_old_pictures_and_restarts_the_delay() { + async fn a_gap_discards_old_pictures_and_restarts_the_delay() { tokio::time::pause(); let delay = Duration::from_secs(2); let media = media(delay); @@ -527,7 +527,7 @@ mod tests { tokio::time::advance(delay).await; tokio::task::yield_now().await; assert_eq!(decoded.lock().unwrap().len(), 1, "old picture held inside decoder"); - producer.discontinuity().unwrap(); + producer.seek(2).unwrap(); producer.write(frame(500, true)).unwrap(); producer.finish().unwrap(); let restarted = Instant::now(); From eb770ba2e0844536ac338d8d57ee69e6bcebe2fe Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:17:26 -0700 Subject: [PATCH 04/22] fix(play): align resumed roles after draining their media Co-Authored-By: GPT-6 --- rs/moq-audio/src/decode/consumer.rs | 63 +++++++ rs/moq-cli/src/play/buffer.rs | 2 + rs/moq-cli/src/play/media.rs | 267 ++++++++++++++++++++++++++- rs/moq-cli/src/play/timeline.rs | 272 +++++++++++++++++++++------- rs/moq-cli/src/play/video.rs | 58 +++++- 5 files changed, 583 insertions(+), 79 deletions(-) diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 2e1694da67a..1711a3c0b14 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -1026,6 +1026,69 @@ mod tests { } } + #[tokio::test] + async fn decoded_segments_keep_resampler_tails_before_fresh_encoder_frames() { + tokio::time::pause(); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 44_100, 1); + let mut consumer = Consumer::new( + &broadcast.consume(), + &catalog, + "audio", + Options { + output: Output { + sample_rate: Some(48_000), + ..Output::default() + }, + max_delay: Duration::from_secs(1), + ..Options::new() + }, + ) + .await + .unwrap(); + for (micros, sample) in [(0, 0.25f32), (30_000, 0.5)] { + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_micros(micros).unwrap(), + duration: Some(Timestamp::from_scale(1024, 44_100).unwrap()), + keyframe: true, + payload: (0..1024).flat_map(|_| sample.to_le_bytes()).collect::>().into(), + }) + .unwrap(); + if micros == 0 { + producer.discontinuity().unwrap(); + } + } + producer.finish().unwrap(); + let mut ended = None; + let mut tail = false; + let mut resumed = false; + while let Some(event) = consumer.read().await.unwrap() { + match event { + Read::End { segment, .. } if ended.is_none() => ended = Some(segment), + Read::Frame { segment, .. } if ended == Some(segment) => { + assert!(!resumed); + tail = true; + } + Read::Frame { segment, .. } if ended.is_some_and(|old| old != segment) => resumed = true, + Read::Gap { .. } => panic!("a declared encoder boundary is not loss"), + _ => {} + } + } + assert!( + tail, + "the old resampler tail must follow its End without looking like a new encoder" + ); + assert!(resumed, "a later group opens a fresh decoded segment"); + } + /// A packet whose sample count isn't a multiple of the resampler's chunk leaves /// samples buffered, and the next output starts with those. Stamping that /// output with the packet that completed the chunk puts it up to a chunk late, diff --git a/rs/moq-cli/src/play/buffer.rs b/rs/moq-cli/src/play/buffer.rs index d03bf3590c4..514fccd92a6 100644 --- a/rs/moq-cli/src/play/buffer.rs +++ b/rs/moq-cli/src/play/buffer.rs @@ -17,6 +17,8 @@ pub(super) struct Buffer { pub floor: Option, waiting_keyframe: bool, pub ended: bool, + /// Exclusive endpoint declared for this encoder segment. + pub end: Option, } impl Buffer { diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 16c3c477d0f..e7c8eb9e378 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -405,11 +405,12 @@ async fn play_audio( // The playout delay is the consumer's to size, from how unevenly packets arrive, // and the sink is where it lives: a sample handed over now sounds that much // later. That estimate times each packet when `read` hands it over, so this - // loop never waits on the speaker between reads. It writes everything as it - // comes and lets `fit` hold the sink on the target instead. The window + // receiver never waits on the speaker between reads. Live playback lets + // `fit` hold the sink on the target; declared ends queue their replacements + // until the old sink drains. The window // schedules video against where the speaker actually is, which keeps the two // together whatever the target does. - let mut role = Role::new(presentation.clone(), changed.clone(), Kind::Audio); + let role = Role::new(presentation.clone(), changed.clone(), Kind::Audio); let received = Mutex::new(VecDeque::new()); let arrived = tokio::sync::Notify::new(); let max_delay = consumer.max_delay(); @@ -448,6 +449,7 @@ async fn play_audio( let mut dropping = false; let receive = async { + let mut end = None; loop { let event = match consumer.read().await { Err(err) if !matches!(err, moq_audio::Error::Decode(_)) => return Err(err.into()), @@ -455,6 +457,25 @@ async fn play_audio( }; let ended = matches!(event, Ok(None)); let delay = consumer.delay(); + match &event { + Ok(Some(moq_audio::decode::Read::End { segment, .. })) => end = Some(*segment), + Ok(Some(moq_audio::decode::Read::Gap { .. })) => end = None, + Ok(Some(moq_audio::decode::Read::Frame { segment, frame })) + if end.is_some_and(|end| end != *segment) => + { + let frame_duration = + Duration::from_secs_f64((frame.data.len() / stride) as f64 / sample_rate as f64); + let first = latency.max( + delay + .min(max_delay) + .max(AUDIO_BUFFER_MIN) + .saturating_sub(frame_duration), + ); + role.plan(frame.timestamp, Some(first)); + end = None; + } + _ => {} + } received.lock().unwrap().push_back((event, delay)); arrived.notify_one(); if ended { @@ -464,6 +485,7 @@ async fn play_audio( }; let play = async { let mut end_segment = None; + let mut sequential = None; loop { let (event, delay) = loop { if let Some(event) = received.lock().unwrap().pop_front() { @@ -476,19 +498,39 @@ async fn play_audio( if end_segment.take_if(|end| *end != segment).is_some() { // A declared boundary preserves the old sink, unlike an undeclared hole. drain(sink).await; - role.resume().await; + let frame_duration = + Duration::from_secs_f64((frame.data.len() / stride) as f64 / sample_rate as f64); + let first = latency.max( + delay + .min(max_delay) + .max(AUDIO_BUFFER_MIN) + .saturating_sub(frame_duration), + ); + role.apply(role.resume(frame.timestamp, Some(first)).await?); + let at = presentation + .lock() + .unwrap() + .due(role.map(frame.timestamp)?) + .and_then(|at| at.checked_sub(first)); + if let Some(at) = at { + tokio::time::sleep_until(at.into()).await; + } + sequential = Some(first); timeline = AudioTimeline::default(); sink = speaker.sink(input.clone())?; dry = true; } frame } - Ok(Some(moq_audio::decode::Read::End { segment, .. })) => { + Ok(Some(moq_audio::decode::Read::End { segment, end })) => { end_segment = Some(segment); + role.end(end)?; continue; } Ok(Some(moq_audio::decode::Read::Gap { .. })) => { end_segment = None; + role.gap(); + sequential = None; continue; } Ok(None) => break, @@ -509,19 +551,34 @@ async fn play_audio( dropping = false; let length = frame.data.len() / stride; - let start = timestamp(frame.timestamp); + let mapped = role.map(frame.timestamp)?; + let start = timestamp(mapped); let timing = timeline.push(start, length, sample_rate, fill_max); // A rewind or a hole too large to fill starts a new playback sink. The old // sink has no media clock, so its buffered audio cannot be carried across a // timeline region the player skipped. if timing.reset_sink { + sequential = None; drop(sink); presentation.lock().unwrap().restarted(); sink = speaker.sink(input.clone())?; dry = true; } + // Resumed media was received while the old tail played. Keep its + // spacing instead of fitting that queued burst as if it arrived live. + if let Some(lead) = sequential { + let at = presentation + .lock() + .unwrap() + .due(mapped) + .and_then(|at| at.checked_sub(lead)); + if let Some(at) = at { + tokio::time::sleep_until(at.into()).await; + } + } + // A hole in the media is a hole in the audio, not a splice. Handing the next // frame straight to the speaker shortens the track by the missing duration, // which leaves it running ahead of media time until the clock below @@ -643,6 +700,204 @@ mod tests { assert!(old.last().unwrap().to <= new.first().unwrap().from); } + #[tokio::test] + async fn paired_quick_resumes_keep_audio_and_reordered_video_together() { + quick_resumes(true, true, Duration::from_millis(200)).await; + } + + #[tokio::test] + async fn an_audio_resume_aligns_with_continuous_video() { + quick_resumes(true, false, Duration::from_millis(200)).await; + } + + #[tokio::test] + async fn a_video_resume_aligns_with_continuous_audio() { + quick_resumes(false, true, Duration::from_millis(200)).await; + } + + #[tokio::test] + async fn paired_resumes_follow_the_speaker_when_delays_differ() { + quick_resumes(true, true, Duration::from_millis(80)).await; + } + + async fn quick_resumes(audio_end: bool, video_end: bool, audio_latency: Duration) { + struct Buffered(Vec); + impl super::super::video::Decoder for Buffered { + async fn decode(&mut self, frame: moq_mux::container::Frame) -> anyhow::Result> { + self.0.push(moq_video::Frame::new( + moq_video::Surface::rgba(&[128; 16 * 16 * 4], moq_video::Size::new(16, 16))?, + frame.timestamp, + )); + self.0.sort_by_key(|frame| frame.timestamp); + let ready = self.0.len().saturating_sub(2); + Ok(self.0.drain(..ready).collect()) + } + async fn flush(&mut self) -> anyhow::Result> { + Ok(std::mem::take(&mut self.0)) + } + } + tokio::time::pause(); + let playback_start = Instant::now(); + let delay = Duration::from_millis(200); + let recorder = Recorder::default(); + let origin = moq_tokio::origin::spawn(); + let media = media(&origin, delay, recorder.clone()); + let broadcast = moq_net::broadcast::Info::new().produce(); + let audio = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let video = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = video + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(delay)) + .await + .unwrap(); + let mut audio = moq_mux::container::Producer::new(audio, Container::Legacy(moq_mux::container::Kind::Audio)); + let mut video = moq_mux::container::Producer::new(video, Container::Legacy(moq_mux::container::Kind::Video)); + let mut options = moq_audio::decode::Options::new(); + options.delay = Some(delay); + options.max_delay = delay; + options.output.format = moq_audio::Format::F32; + let consumer = moq_audio::decode::Consumer::new( + &broadcast.consume(), + &AudioConfig::new(AudioCodec::Pcm, SAMPLE_RATE, 1), + "audio", + options, + ) + .await + .unwrap(); + let audio_play = AudioPlayback { + changed: media.drained.clone(), + speaker: recorder.clone(), + presentation: media.presentation.clone(), + latency: audio_latency, + output: recorder.clone(), + }; + let video_play = Video { + presentation: media.presentation.clone(), + frames: media.video.clone(), + changed: media.drained.clone(), + output: recorder.clone(), + max_delay: delay, + }; + let audio_task = tokio::spawn(async move { + play_audio(consumer, audio_play).await.unwrap().finish().await; + }); + let video_task = tokio::spawn(video_play.run( + moq_mux::container::Consumer::new(subscriber, Container::Legacy(moq_mux::container::Kind::Video)), + Buffered(Vec::new()), + )); + let picture = |index: u64| moq_mux::container::Frame { + timestamp: moq_net::Timestamp::from_millis(index * 20).unwrap(), + duration: Some(moq_net::Timestamp::from_millis(20).unwrap()), + keyframe: index == 0, + payload: Bytes::from_static(b"picture"), + }; + for index in 0..3 { + let mut pcm = packet(index, 0.25); + pcm.keyframe = index == 0; + audio.write(pcm).unwrap(); + video.write(picture(index)).unwrap(); + tokio::time::sleep(PACKET_DURATION).await; + } + if audio_end { + audio.discontinuity().unwrap(); + } + if video_end { + video.discontinuity().unwrap(); + } + for index in 0..3 { + if audio_end || index == 0 { + let mut pcm = packet(if audio_end { index } else { index + 3 }, 0.5); + pcm.keyframe = index == 0; + audio.write(pcm).unwrap(); + } + if video_end || index == 0 { + video.write(picture(if video_end { index } else { index + 3 })).unwrap(); + } + } + if audio_end { + audio.finish().unwrap(); + } + if video_end { + video.finish().unwrap(); + } + let mut shown = Vec::new(); + for step in 0..800 { + if step >= 20 && step % 20 == 0 { + let index = 3 + step / 20; + if !audio_end { + audio.write(packet(index, 0.5)).unwrap(); + } + if !video_end { + video.write(picture(index)).unwrap(); + } + } + tokio::task::yield_now().await; + if let Some(timestamp) = recorder.present(&media) { + shown.push((timestamp, Instant::now())); + } + tokio::time::advance(Duration::from_millis(1)).await; + } + if !audio_end { + audio.finish().unwrap(); + } + if !video_end { + video.finish().unwrap(); + } + audio_task.await.unwrap(); + video_task.await.unwrap().unwrap(); + if video_end { + assert_eq!(shown.len(), 6, "both decoder tails must be shown"); + } + assert_eq!( + shown[..3] + .iter() + .map(|(timestamp, _)| timestamp.as_millis()) + .collect::>(), + [0, 20, 40] + ); + let played = recorder.played(); + let old: Vec<_> = played.iter().filter(|part| part.sample == 0.25).collect(); + let new: Vec<_> = played.iter().filter(|part| part.sample == 0.5).collect(); + assert_eq!( + old.iter().map(|part| part.to - part.from).sum::(), + PACKET_DURATION * 3 + ); + if audio_end { + assert_eq!( + new.iter().map(|part| part.to - part.from).sum::(), + PACKET_DURATION * 3 + ); + } + assert!(old.last().unwrap().to <= new[0].from); + let (picture, sample) = if audio_end && video_end { + (shown[3].1, new[0].from) + } else if audio_end { + let timestamp = new[0].from.duration_since(playback_start + delay); + let shown = shown + .iter() + .find(|(shown, _)| { + let start = super::timestamp(*shown); + start <= timestamp && timestamp < start + PACKET_DURATION + }) + .expect("continuous video reaches resumed audio"); + (shown.1 + (timestamp - super::timestamp(shown.0)), new[0].from) + } else { + let timestamp = super::timestamp(shown[3].0); + let index = (shown[3].0.as_millis() / 20) as usize; + let offset = timestamp - PACKET_DURATION * index as u32; + (shown[3].1, new[index - 3].from + offset) + }; + assert!( + picture.saturating_duration_since(sample) <= Duration::from_millis(1) + && sample.saturating_duration_since(picture) <= Duration::from_millis(1), + "new A/V clocks diverged: picture={picture:?} sample={sample:?}" + ); + } + /// A mono PCM rendition, published and named in the catalog until dropped. fn rendition( broadcast: &moq_net::broadcast::Producer, diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index ba8644bdf12..57ec295e32a 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -1,5 +1,6 @@ //! Media time: the playout clock, and the audio track's own timeline. +use anyhow::Context; use std::time::{Duration, Instant}; use super::playback::Kind; @@ -29,9 +30,27 @@ pub(super) struct Presentation { /// audio frame pins the anchor outright instead of pacing against media it /// will never play. restart: bool, - /// Segment reached by each active playback role. - roles: [Option; 2], - segment: u64, + /// Endpoints declared by each role, on the shared playback timeline. + roles: [Segment; 2], + reference: Option, +} + +#[derive(Default)] +struct Segment { + end: Option, + plans: std::collections::VecDeque, + active: Option, +} + +struct Boundary { + starts: Option, + end: Duration, +} + +struct Plan { + source: Duration, + delay: Duration, + mapping: Option, } impl Presentation { @@ -41,32 +60,21 @@ impl Presentation { delay, speaker: false, restart: false, - roles: [None; 2], - segment: 0, + roles: Default::default(), + reference: None, } } - pub(super) fn started(&mut self, role: Kind) { - self.roles[role as usize] = Some(0); - } - - pub(super) fn finished(&mut self, role: Kind) { - self.roles[role as usize] = None; - } - - /// Rebase once every active role has played out the previous segment. - pub(super) fn resume(&mut self, role: Kind, segment: u64) -> bool { - self.roles[role as usize] = Some(segment); - if self.roles.iter().flatten().any(|ready| *ready < segment) { - return false; - } - if self.segment < segment { - self.segment = segment; - self.pacer = moq_mux::Pacer::default().with_delay(self.delay); - self.speaker = false; - self.restart = false; - } - true + /// The media position the shared clock will reach at `at`. + fn position(&self, at: Instant) -> Option { + let reference = self.reference?; + let due = self.due(reference)?; + let reference = timestamp(reference); + Some(if at >= due { + reference.saturating_add(at.duration_since(due)) + } else { + reference.saturating_sub(due.duration_since(at)) + }) } /// Fold an arriving video frame into the anchor, unless the speaker owns it, @@ -79,6 +87,7 @@ impl Presentation { return false; } + self.reference = Some(timestamp); let before = self.pacer.due(timestamp); before != Some(self.pacer.pace(timestamp, now)) } @@ -104,6 +113,7 @@ impl Presentation { /// recompute the deadline it is already asleep on. pub(super) fn audio(&mut self, end: Duration, buffered: Duration, now: Instant) -> bool { let timestamp = media(end); + self.reference = Some(timestamp); let edge = now .checked_add(buffered) .and_then(|at| at.checked_sub(self.delay)) @@ -152,12 +162,35 @@ impl Presentation { } } -/// Releases a role when its track ends or the task is cancelled. +/// Maps each encoder segment onto the shared playback clock. pub(super) struct Role { presentation: std::sync::Arc>, changed: std::sync::Arc, role: Kind, - segment: u64, + mapping: std::sync::Mutex, +} + +#[derive(Clone, Copy, Default)] +pub(super) struct Mapping { + source: Duration, + target: Duration, + starts: Option, +} + +impl Mapping { + fn map(self, source: Timestamp) -> anyhow::Result { + let source = Duration::from(source); + let target = if source >= self.source { + self.target + .checked_add(source - self.source) + .context("playback timeline overflow")? + } else { + self.target + .checked_sub(self.source - source) + .context("picture precedes the playback timeline")? + }; + Ok(Timestamp::from_micros(u64::try_from(target.as_micros())?)?) + } } impl Role { @@ -166,29 +199,120 @@ impl Role { changed: std::sync::Arc, role: Kind, ) -> Self { - presentation.lock().unwrap().started(role); Self { presentation, changed, role, - segment: 0, + mapping: Default::default(), } } - pub(super) async fn resume(&mut self) { - self.segment += 1; - self.presentation.lock().unwrap().resume(self.role, self.segment); + pub(super) fn map(&self, source: Timestamp) -> anyhow::Result { + self.mapping.lock().unwrap().map(source) + } + + pub(super) fn end(&self, source: Timestamp) -> anyhow::Result<()> { + let mapping = *self.mapping.lock().unwrap(); + self.presentation.lock().unwrap().roles[self.role as usize].end = Some(Boundary { + starts: mapping.starts, + end: timestamp(mapping.map(source)?), + }); self.changed.notify_waiters(); + Ok(()) + } + + pub(super) fn plan(&self, source: Timestamp, delay: Option) { + let mut presentation = self.presentation.lock().unwrap(); + let delay = delay.unwrap_or(presentation.delay); + presentation.roles[self.role as usize].plans.push_back(Plan { + source: Duration::from(source), + delay, + mapping: None, + }); + self.changed.notify_waiters(); + } + + pub(super) fn gap(&self) { + let mut presentation = self.presentation.lock().unwrap(); + presentation.roles[self.role as usize].end = None; + presentation.roles[self.role as usize].plans.clear(); + presentation.roles[self.role as usize].active = None; + self.changed.notify_waiters(); + } + + pub(super) fn apply(&self, mapping: Mapping) { + *self.mapping.lock().unwrap() = mapping; + let mut presentation = self.presentation.lock().unwrap(); + presentation.roles[self.role as usize].active = Some(mapping); + presentation.roles[self.role as usize].plans.pop_front(); + self.changed.notify_waiters(); + } + + /// Drain declared old media, then align the new encoder to the running clock. + pub(super) async fn resume(&self, source: Timestamp, delay: Option) -> anyhow::Result { loop { - let changed = self.changed.notified(); - // Register before inspecting shared state so a peer cannot finish between them. - tokio::pin!(changed); - changed.as_mut().enable(); - if self.presentation.lock().unwrap().resume(self.role, self.segment) { - self.changed.notify_waiters(); - return; + let now = tokio::time::Instant::now().into_std(); + let end = { + let presentation = self.presentation.lock().unwrap(); + presentation + .roles + .iter() + .filter_map(|role| role.end.as_ref()) + .filter(|boundary| boundary.starts.is_none_or(|start| start <= now)) + .filter_map(|boundary| presentation.due(media(boundary.end))) + .max() + }; + if let Some(end) = end.filter(|end| *end > now) { + tokio::select! { + _ = tokio::time::sleep_until(end.into()) => {}, + _ = self.changed.notified() => {}, + } + continue; + } + let mut presentation = self.presentation.lock().unwrap(); + let own = self.role as usize; + let peer = 1 - own; + if let Some(mapping) = presentation.roles[own].plans.front().and_then(|plan| plan.mapping) { + return Ok(mapping); } - changed.await; + // Both roles already have replacement media. The speaker's first sample + // defines their common origin even when its adaptive delay differs from video's. + if presentation.roles[own] + .plans + .front() + .is_some_and(|plan| plan.mapping.is_none()) + && presentation.roles[peer] + .plans + .front() + .is_some_and(|plan| plan.mapping.is_none()) + { + let audio = presentation.roles[Kind::Audio as usize].plans.front().unwrap(); + let starts = now + .checked_add(audio.delay) + .context("playback delay exceeds the system clock")?; + let mapping = Mapping { + source: audio.source, + target: presentation.position(starts).unwrap_or(audio.source), + starts: Some(starts), + }; + presentation.roles[own].plans.front_mut().unwrap().mapping = Some(mapping); + presentation.roles[peer].plans.front_mut().unwrap().mapping = Some(mapping); + return Ok(mapping); + } + if let Some(mapping) = presentation.roles[peer] + .active + .filter(|mapping| mapping.starts.is_some_and(|starts| starts > now)) + { + return Ok(mapping); + } + let starts = now + .checked_add(delay.unwrap_or(presentation.delay)) + .context("playback delay exceeds the system clock")?; + return Ok(Mapping { + source: Duration::from(source), + target: presentation.position(starts).unwrap_or_else(|| timestamp(source)), + starts: Some(starts), + }); } } } @@ -196,7 +320,9 @@ impl Role { impl Drop for Role { fn drop(&mut self) { if let Ok(mut presentation) = self.presentation.lock() { - presentation.finished(self.role); + presentation.roles[self.role as usize].end = None; + presentation.roles[self.role as usize].plans.clear(); + presentation.roles[self.role as usize].active = None; } self.changed.notify_waiters(); } @@ -317,30 +443,48 @@ mod tests { Timestamp::from_millis(millis).unwrap() } - #[test] - fn a_segment_rebases_only_after_both_playback_roles_drain() { - let start = Instant::now(); - let mut presentation = Presentation::new(DELAY); - presentation.started(Kind::Video); - presentation.started(Kind::Audio); - presentation.audio(Duration::from_secs(10), DELAY, start); - let old = presentation.due(ms(10_000)); - assert!(!presentation.resume(Kind::Video, 1)); - assert_eq!(presentation.due(ms(10_000)), old); - assert!(presentation.resume(Kind::Audio, 1)); - assert_eq!(presentation.due(ms(10_000)), None); - presentation.audio(Duration::from_secs(1), DELAY, start); - assert!(presentation.resume(Kind::Video, 1)); + #[tokio::test] + async fn one_ended_role_maps_to_the_continuous_peers_clock() { + tokio::time::pause(); + let start = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); + let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + presentation.lock().unwrap().video(ms(10_000), start.into_std()); + audio.end(ms(10_020)).unwrap(); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + let unchanged = presentation.lock().unwrap().due(ms(10_500)); + audio.apply(audio.resume(ms(0), Some(DELAY)).await.unwrap()); + assert_eq!(audio.map(ms(0)).unwrap().as_millis(), 10_120); + assert_eq!(video.map(ms(10_500)).unwrap().as_millis(), 10_500); + assert_eq!(presentation.lock().unwrap().due(ms(10_500)), unchanged); + } + + #[tokio::test] + async fn paired_plans_keep_media_offsets_and_follow_the_speakers_delay() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); + let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + audio.plan(ms(0), Some(Duration::from_millis(80))); + video.plan(ms(40), None); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + video.apply(video.resume(ms(40), None).await.unwrap()); + audio.apply(audio.resume(ms(0), Some(Duration::from_millis(80))).await.unwrap()); assert_eq!( - presentation.due(ms(1_000)), - Some(start + DELAY), - "one rebase per segment" + video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + 40 ); - assert!(!presentation.resume(Kind::Audio, 2)); - presentation.finished(Kind::Video); - assert!( - presentation.resume(Kind::Audio, 2), - "a closed track cannot hold its peer forever" + assert_eq!( + presentation.lock().unwrap().due(audio.map(ms(0)).unwrap()), + Some((now + Duration::from_millis(200)).into_std()) ); } diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index b10f547009e..9568826b1e2 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -73,8 +73,10 @@ impl Decoder for moq_video::decode::Sink { impl Video { pub async fn run(self, mut track: Track, mut decoder: impl Decoder) -> anyhow::Result<()> { let closed = std::sync::atomic::AtomicBool::new(false); + let advancing = std::sync::atomic::AtomicBool::new(false); + let revision = std::sync::atomic::AtomicU64::new(0); let buffers = Mutex::new(VecDeque::from([Buffer::default()])); - let mut role = Role::new( + let role = Role::new( self.presentation.clone(), self.changed.clone(), super::playback::Kind::Video, @@ -87,11 +89,12 @@ impl Video { loop { let read = track.read().await?; let mut buffers = buffers.lock().unwrap(); - let active = buffers.len() == 1; + let active = buffers.len() == 1 && !advancing.load(std::sync::atomic::Ordering::Relaxed); let before = buffers.front().unwrap().generation; let ended = match read { Some(Read::Frame { group, frame }) => { if end.take_if(|end| *end != group).is_some() { + role.plan(frame.timestamp, None); buffers.push_back(Buffer::default()); } if active @@ -99,7 +102,7 @@ impl Video { .presentation .lock() .unwrap() - .video(frame.timestamp, Instant::now().into_std()) + .video(role.map(frame.timestamp)?, Instant::now().into_std()) { self.output.send(Event::Wake); } @@ -109,6 +112,8 @@ impl Video { // Applied in delivery order, so the restart lands before the frame after it. Some(Read::Gap { .. }) => { end = None; + revision.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + role.gap(); buffers.clear(); buffers.push_back(Buffer::default()); buffers.front_mut().unwrap().clear(); @@ -116,9 +121,13 @@ impl Video { self.presentation.lock().unwrap().video_restarted(); false } - Some(Read::End { group, .. }) => { + Some(Read::End { group, end: endpoint }) => { end = Some(group); buffers.back_mut().unwrap().ended = true; + buffers.back_mut().unwrap().end = Some(endpoint); + if active { + role.end(endpoint)?; + } false } None => { @@ -187,6 +196,7 @@ impl Video { // Every access unit up to the earliest picture still owed must be // decoded before that picture is due, however deep the stream // reorders or the codec holds pictures back. + let oldest = role.map(oldest)?; let owed = held.first().map_or(oldest, |held| oldest.min(*held)); let at = presentation .due(owed) @@ -229,13 +239,14 @@ impl Video { continue; } Step::Decode => { - let frame = buffers + let mut frame = buffers .lock() .unwrap() .front_mut() .unwrap() .pop() .expect("no await since inspecting the buffer"); + frame.timestamp = role.map(frame.timestamp)?; generation = Some(current); held.insert(frame.timestamp); // Never race a codec call: cancelling Sink::decode poisons it. The @@ -246,6 +257,7 @@ impl Video { let next = { let mut buffers = buffers.lock().unwrap(); if buffers.len() > 1 { + advancing.store(true, std::sync::atomic::Ordering::Relaxed); buffers.pop_front(); true } else { @@ -259,11 +271,28 @@ impl Video { self.changed.notified().await; continue; } - role.resume().await; + let start = buffers + .lock() + .unwrap() + .front() + .unwrap() + .oldest() + .expect("a resumed group has a frame"); + let before = revision.load(std::sync::atomic::Ordering::Relaxed); + let mapping = role.resume(start, None).await?; + if revision.load(std::sync::atomic::Ordering::Relaxed) != before { + advancing.store(false, std::sync::atomic::Ordering::Relaxed); + continue; + } + role.apply(mapping); let buffers = buffers.lock().unwrap(); + if let Some(end) = buffers.front().unwrap().end { + role.end(end)?; + } + advancing.store(false, std::sync::atomic::Ordering::Relaxed); let mut presentation = self.presentation.lock().unwrap(); for frame in &buffers.front().unwrap().frames { - presentation.video(frame.timestamp, Instant::now().into_std()); + presentation.video(role.map(frame.timestamp)?, Instant::now().into_std()); } self.output.send(Event::Wake); continue; @@ -281,7 +310,14 @@ impl Video { } // A codec may return pictures it held across the keyframe that // resumed a skipped timeline. Those pictures no longer have a slot. - let floor = buffers.lock().unwrap().front().unwrap().floor; + let floor = buffers + .lock() + .unwrap() + .front() + .unwrap() + .floor + .map(|floor| role.map(floor)) + .transpose()?; pending.extend( frames .into_iter() @@ -636,7 +672,11 @@ mod tests { tokio::time::advance(Duration::from_millis(1)).await; } task.await.unwrap().unwrap(); - assert_eq!(shown, [0, 33, 66, 0, 33, 66]); + assert_eq!(&shown[..3], [0, 33, 66]); + assert_eq!(shown.len(), 6); + assert!(shown[3] > shown[2], "the resumed segment follows every old picture"); + assert_eq!(shown[4] - shown[3], 33); + assert_eq!(shown[5] - shown[4], 33); assert_eq!( *flushed.lock().unwrap(), 2, From 6abcc73f05c42f02170ed490decc0383dee38363 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 21:40:48 -0700 Subject: [PATCH 05/22] chore(quest): complete sequential playback resume Co-Authored-By: GPT-6 --- quest/m2/README.md | 1 - quest/m2/play-resume.md | 31 ------------------------------- 2 files changed, 32 deletions(-) delete mode 100644 quest/m2/play-resume.md diff --git a/quest/m2/README.md b/quest/m2/README.md index 221d8bdad6f..64623b78923 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -166,5 +166,4 @@ ranked just below deterministic TS legs, which it requires. - [Publisher timeliness at relay ingest](/quest/m2/qos-publisher-timeliness.md) - follow-on capability after the release contract is stable - [Release execution](/quest/m2/release/README.md) - publication, installed-artifact proof, cleanup, and downstream adoption after m1 -- [Play resume](/quest/m2/play-resume.md) - `moq-cli play` plays out pictures buffered before a declared end when new media arrives quickly - [Task supervision](/quest/m2/task-supervision.md) - the auth decider loop, stats publisher, and moq-tokio's token and WebSocket internals are supervised like #5447's tasks diff --git a/quest/m2/play-resume.md b/quest/m2/play-resume.md deleted file mode 100644 index 7a2b63dc80c..00000000000 --- a/quest/m2/play-resume.md +++ /dev/null @@ -1,31 +0,0 @@ -# [M] moq-cli play plays out pictures before a quick resume - -## Goal - -When new media arrives before the pictures buffered ahead of a declared end -have played (a source switch, say), `moq-cli play` plays the old timeline out -first instead of dropping it. - -## Plan - -#5288 fixed `rs/moq-cli/src/play/video.rs` so `Read::End` no longer discards -the buffered pictures, with the restart waiting for the first frame of a later -group. A resume that arrives before those pictures play still drops them, as -it did before #5288. - -Playing them out needs a sequential timeline in `play`: queue the new group's -frames behind the remaining pre-end pictures, then rebase the clock at the -boundary. Queueing encoded frames is not enough when the decoder reorders: the -declared end has to reach the decode loop so it flushes the pictures it still -holds before decoding the resumed group, which today happens only when the -track ends. - -✅ Decided 2026-10-13: audio queues with video. One sequential timeline: audio -and video both play out the old segment, then rebase together, which keeps -A/V sync. Not chosen: audio snaps to the new timeline; leave it open. - -Test with a declared end followed immediately by a new group, using a decoder -that holds pictures (depth above zero): every pre-end picture is shown before -the first new one. - -Public API: none. Wire: none. From c0c1db3bf0deafb1f2b2cac368f5041f1f282655 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 11:33:25 -0700 Subject: [PATCH 06/22] fix(play): pace resumed audio against the current playout target Sequential pacing after a clean resume held every later frame to the lead computed from the first resumed frame, so a growing jitter target could not deepen the sink. Recompute the lead per frame from the consumer's current delay, sharing one helper with the resume plan. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/media.rs | 39 +++++++++++++++--------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index e7c8eb9e378..1fb6b3aa922 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -444,6 +444,13 @@ async fn play_audio( let mut timeline = AudioTimeline::default(); + // How far ahead of its due time a frame is written: the playout target less + // the frame itself, never under the sink's floor. + let lead = |delay: Duration, frame: &moq_audio::Frame| { + let duration = Duration::from_secs_f64((frame.data.len() / stride) as f64 / sample_rate as f64); + latency.max(delay.min(max_delay).max(AUDIO_BUFFER_MIN).saturating_sub(duration)) + }; + // Tracks whether the last read failed, so a stream the decoder can't read at // all logs once rather than once per packet. let mut dropping = false; @@ -463,15 +470,7 @@ async fn play_audio( Ok(Some(moq_audio::decode::Read::Frame { segment, frame })) if end.is_some_and(|end| end != *segment) => { - let frame_duration = - Duration::from_secs_f64((frame.data.len() / stride) as f64 / sample_rate as f64); - let first = latency.max( - delay - .min(max_delay) - .max(AUDIO_BUFFER_MIN) - .saturating_sub(frame_duration), - ); - role.plan(frame.timestamp, Some(first)); + role.plan(frame.timestamp, Some(lead(delay, frame))); end = None; } _ => {} @@ -485,7 +484,7 @@ async fn play_audio( }; let play = async { let mut end_segment = None; - let mut sequential = None; + let mut sequential = false; loop { let (event, delay) = loop { if let Some(event) = received.lock().unwrap().pop_front() { @@ -498,14 +497,7 @@ async fn play_audio( if end_segment.take_if(|end| *end != segment).is_some() { // A declared boundary preserves the old sink, unlike an undeclared hole. drain(sink).await; - let frame_duration = - Duration::from_secs_f64((frame.data.len() / stride) as f64 / sample_rate as f64); - let first = latency.max( - delay - .min(max_delay) - .max(AUDIO_BUFFER_MIN) - .saturating_sub(frame_duration), - ); + let first = lead(delay, &frame); role.apply(role.resume(frame.timestamp, Some(first)).await?); let at = presentation .lock() @@ -515,7 +507,7 @@ async fn play_audio( if let Some(at) = at { tokio::time::sleep_until(at.into()).await; } - sequential = Some(first); + sequential = true; timeline = AudioTimeline::default(); sink = speaker.sink(input.clone())?; dry = true; @@ -530,7 +522,7 @@ async fn play_audio( Ok(Some(moq_audio::decode::Read::Gap { .. })) => { end_segment = None; role.gap(); - sequential = None; + sequential = false; continue; } Ok(None) => break, @@ -559,7 +551,7 @@ async fn play_audio( // sink has no media clock, so its buffered audio cannot be carried across a // timeline region the player skipped. if timing.reset_sink { - sequential = None; + sequential = false; drop(sink); presentation.lock().unwrap().restarted(); sink = speaker.sink(input.clone())?; @@ -568,12 +560,13 @@ async fn play_audio( // Resumed media was received while the old tail played. Keep its // spacing instead of fitting that queued burst as if it arrived live. - if let Some(lead) = sequential { + // The lead follows the current target so jitter can still deepen the sink. + if sequential { let at = presentation .lock() .unwrap() .due(mapped) - .and_then(|at| at.checked_sub(lead)); + .and_then(|at| at.checked_sub(lead(delay, &frame))); if let Some(at) = at { tokio::time::sleep_until(at.into()).await; } From dcaaddbdd2281a90198652217d27004e7dbd743a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 11:48:17 -0700 Subject: [PATCH 07/22] fix(play): keep a late paired replacement on its peer's mapping When both roles declared an end but one replacement arrived after the peer's resumed media started, the late role fell through to an independent mapping and lost the source A/V offset. Record the peer's replacement mapping while a role's end is pending, and resume on it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/timeline.rs | 67 ++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 14 deletions(-) diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index 57ec295e32a..433a5ccb14b 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -38,6 +38,10 @@ pub(super) struct Presentation { #[derive(Default)] struct Segment { end: Option, + /// Whether a declared end is still waiting for its replacement. + pending: bool, + /// The peer's replacement mapping, applied while this role was pending. + paired: Option, plans: std::collections::VecDeque, active: Option, } @@ -213,10 +217,14 @@ impl Role { pub(super) fn end(&self, source: Timestamp) -> anyhow::Result<()> { let mapping = *self.mapping.lock().unwrap(); - self.presentation.lock().unwrap().roles[self.role as usize].end = Some(Boundary { + let mut presentation = self.presentation.lock().unwrap(); + let segment = &mut presentation.roles[self.role as usize]; + segment.end = Some(Boundary { starts: mapping.starts, end: timestamp(mapping.map(source)?), }); + segment.pending = true; + drop(presentation); self.changed.notify_waiters(); Ok(()) } @@ -233,18 +241,24 @@ impl Role { } pub(super) fn gap(&self) { - let mut presentation = self.presentation.lock().unwrap(); - presentation.roles[self.role as usize].end = None; - presentation.roles[self.role as usize].plans.clear(); - presentation.roles[self.role as usize].active = None; + self.presentation.lock().unwrap().roles[self.role as usize] = Segment::default(); self.changed.notify_waiters(); } pub(super) fn apply(&self, mapping: Mapping) { *self.mapping.lock().unwrap() = mapping; let mut presentation = self.presentation.lock().unwrap(); - presentation.roles[self.role as usize].active = Some(mapping); - presentation.roles[self.role as usize].plans.pop_front(); + let own = &mut presentation.roles[self.role as usize]; + own.active = Some(mapping); + own.plans.pop_front(); + own.pending = false; + own.paired = None; + // A peer that ended too resumes on this mapping, however late its media lands. + let peer = &mut presentation.roles[1 - self.role as usize]; + if peer.pending { + peer.paired = Some(mapping); + } + drop(presentation); self.changed.notify_waiters(); } @@ -299,10 +313,11 @@ impl Role { presentation.roles[peer].plans.front_mut().unwrap().mapping = Some(mapping); return Ok(mapping); } - if let Some(mapping) = presentation.roles[peer] - .active - .filter(|mapping| mapping.starts.is_some_and(|starts| starts > now)) - { + if let Some(mapping) = presentation.roles[own].paired.or_else(|| { + presentation.roles[peer] + .active + .filter(|mapping| mapping.starts.is_some_and(|starts| starts > now)) + }) { return Ok(mapping); } let starts = now @@ -320,9 +335,7 @@ impl Role { impl Drop for Role { fn drop(&mut self) { if let Ok(mut presentation) = self.presentation.lock() { - presentation.roles[self.role as usize].end = None; - presentation.roles[self.role as usize].plans.clear(); - presentation.roles[self.role as usize].active = None; + presentation.roles[self.role as usize] = Segment::default(); } self.changed.notify_waiters(); } @@ -460,6 +473,32 @@ mod tests { assert_eq!(presentation.lock().unwrap().due(ms(10_500)), unchanged); } + #[tokio::test] + async fn a_late_paired_replacement_keeps_its_media_offset() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); + let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + audio.plan(ms(0), Some(DELAY)); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + audio.apply(audio.resume(ms(0), Some(DELAY)).await.unwrap()); + // The video replacement lands after the resumed audio has started playing. + tokio::time::advance(DELAY * 2).await; + video.plan(ms(40), None); + video.apply(video.resume(ms(40), None).await.unwrap()); + assert_eq!( + video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + 40 + ); + } + #[tokio::test] async fn paired_plans_keep_media_offsets_and_follow_the_speakers_delay() { tokio::time::pause(); From 41b64eb9343102f45ffd26eb7b7a5c4c1783afae Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 12:01:08 -0700 Subject: [PATCH 08/22] fix(play): resume audio on any segment change, deferring early ends A resumed segment short enough to declare its end before its first PCM overwrote the pending transition, so its PCM played on the previous segment's mapping and sink. Treat any decoder segment change as the resume, since only a declared end or a gap changes it, and map an end declared for a not-yet-resumed segment once that segment resumes. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/media.rs | 40 ++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 1fb6b3aa922..3158b740892 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -455,8 +455,11 @@ async fn play_audio( // all logs once rather than once per packet. let mut dropping = false; + // A decoder segment only changes at a declared end or a gap, and a gap + // resets this, so any change of segment is a resume. A short segment can + // declare its end before its first PCM, so the end alone cannot mark it. let receive = async { - let mut end = None; + let mut planned = None; loop { let event = match consumer.read().await { Err(err) if !matches!(err, moq_audio::Error::Decode(_)) => return Err(err.into()), @@ -465,13 +468,12 @@ async fn play_audio( let ended = matches!(event, Ok(None)); let delay = consumer.delay(); match &event { - Ok(Some(moq_audio::decode::Read::End { segment, .. })) => end = Some(*segment), - Ok(Some(moq_audio::decode::Read::Gap { .. })) => end = None, - Ok(Some(moq_audio::decode::Read::Frame { segment, frame })) - if end.is_some_and(|end| end != *segment) => - { - role.plan(frame.timestamp, Some(lead(delay, frame))); - end = None; + Ok(Some(moq_audio::decode::Read::Gap { .. })) => planned = None, + Ok(Some(moq_audio::decode::Read::Frame { segment, frame })) => { + let resumed = planned.replace(*segment).is_some_and(|planned| planned != *segment); + if resumed { + role.plan(frame.timestamp, Some(lead(delay, frame))); + } } _ => {} } @@ -483,7 +485,10 @@ async fn play_audio( } }; let play = async { - let mut end_segment = None; + // The segment the role's mapping serves, and an end declared for a later + // one, which can only be mapped once that segment resumes. + let mut mapped = None; + let mut deferred = None; let mut sequential = false; loop { let (event, delay) = loop { @@ -494,7 +499,7 @@ async fn play_audio( }; let frame = match event { Ok(Some(moq_audio::decode::Read::Frame { segment, frame })) => { - if end_segment.take_if(|end| *end != segment).is_some() { + if mapped.replace(segment).is_some_and(|mapped| mapped != segment) { // A declared boundary preserves the old sink, unlike an undeclared hole. drain(sink).await; let first = lead(delay, &frame); @@ -507,6 +512,11 @@ async fn play_audio( if let Some(at) = at { tokio::time::sleep_until(at.into()).await; } + if let Some((ended, end)) = deferred.take() + && ended == segment + { + role.end(end)?; + } sequential = true; timeline = AudioTimeline::default(); sink = speaker.sink(input.clone())?; @@ -515,12 +525,16 @@ async fn play_audio( frame } Ok(Some(moq_audio::decode::Read::End { segment, end })) => { - end_segment = Some(segment); - role.end(end)?; + if mapped.is_none_or(|mapped| mapped == segment) { + role.end(end)?; + } else { + deferred = Some((segment, end)); + } continue; } Ok(Some(moq_audio::decode::Read::Gap { .. })) => { - end_segment = None; + mapped = None; + deferred = None; role.gap(); sequential = false; continue; From 6076917c18f69f1e244c2120152c69a362cc58e5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 12:09:18 -0700 Subject: [PATCH 09/22] fix(play): keep a closed role's declared end for its peer Dropping a role after it declared an end cleared that boundary while the returned sink still played the tail, so the peer's replacement could start early. Keep the end on drop; once it is past, nothing waits on it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/timeline.rs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index 433a5ccb14b..a8aa0121154 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -335,7 +335,13 @@ impl Role { impl Drop for Role { fn drop(&mut self) { if let Ok(mut presentation) = self.presentation.lock() { - presentation.roles[self.role as usize] = Segment::default(); + // A declared end stays: the sink handed back still plays the tail up to it, + // and once it is past, nothing waits on it. + let segment = &mut presentation.roles[self.role as usize]; + *segment = Segment { + end: segment.end.take(), + ..Default::default() + }; } self.changed.notify_waiters(); } @@ -499,6 +505,26 @@ mod tests { ); } + #[tokio::test] + async fn a_closed_role_still_holds_its_declared_end() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); + let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + audio.end(ms(10_200)).unwrap(); + // The track closed, but its returned sink still plays the tail up to the end. + drop(audio); + video.end(ms(10_000)).unwrap(); + video.plan(ms(0), None); + let resume = tokio::time::timeout(DELAY + Duration::from_millis(150), video.resume(ms(0), None)).await; + assert!(resume.is_err(), "video resumed before the audio tail played"); + } + #[tokio::test] async fn paired_plans_keep_media_offsets_and_follow_the_speakers_delay() { tokio::time::pause(); From d5b00ee28f64ee73dd390dc0e53db282ee392990 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 12:18:49 -0700 Subject: [PATCH 10/22] fix(play): close a video resume race and pair by transition A gap landing between popping the old video buffer and reading the next one's start left an empty buffer and panicked. Read the start and revision under the pop's lock, and check the revision under the lock that applies the mapping. A role finishing a transition its peer already resumed now takes that retained mapping before pairing front plans, which may belong to the peer's next transition. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/timeline.rs | 44 ++++++++++++++++++++++++++++----- rs/moq-cli/src/play/video.rs | 21 ++++++---------- 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index a8aa0121154..a1a27d406a9 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -286,7 +286,13 @@ impl Role { let mut presentation = self.presentation.lock().unwrap(); let own = self.role as usize; let peer = 1 - own; - if let Some(mapping) = presentation.roles[own].plans.front().and_then(|plan| plan.mapping) { + // The peer already resumed the transition this role is finishing. + if let Some(mapping) = presentation.roles[own] + .plans + .front() + .and_then(|plan| plan.mapping) + .or(presentation.roles[own].paired) + { return Ok(mapping); } // Both roles already have replacement media. The speaker's first sample @@ -313,11 +319,10 @@ impl Role { presentation.roles[peer].plans.front_mut().unwrap().mapping = Some(mapping); return Ok(mapping); } - if let Some(mapping) = presentation.roles[own].paired.or_else(|| { - presentation.roles[peer] - .active - .filter(|mapping| mapping.starts.is_some_and(|starts| starts > now)) - }) { + if let Some(mapping) = presentation.roles[peer] + .active + .filter(|mapping| mapping.starts.is_some_and(|starts| starts > now)) + { return Ok(mapping); } let starts = now @@ -505,6 +510,33 @@ mod tests { ); } + #[tokio::test] + async fn a_late_replacement_pairs_with_its_own_transition() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); + let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + video.plan(ms(40), None); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + video.apply(video.resume(ms(40), None).await.unwrap()); + // Video already queued its next replacement when audio's first one lands. + video.end(ms(60)).unwrap(); + video.plan(ms(5_000), None); + audio.plan(ms(0), Some(DELAY)); + audio.apply(audio.resume(ms(0), Some(DELAY)).await.unwrap()); + assert_eq!( + video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + 40 + ); + } + #[tokio::test] async fn a_closed_role_still_holds_its_declared_end() { tokio::time::pause(); diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 9568826b1e2..406563f5ed9 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -254,38 +254,33 @@ impl Video { decoder.decode(frame).await? } Step::Resume => { + // Read under the same lock as the pop: a gap replaces the buffers and + // bumps the revision while holding it. let next = { let mut buffers = buffers.lock().unwrap(); if buffers.len() > 1 { advancing.store(true, std::sync::atomic::Ordering::Relaxed); buffers.pop_front(); - true + let start = buffers.front().unwrap().oldest().expect("a resumed group has a frame"); + Some((start, revision.load(std::sync::atomic::Ordering::Relaxed))) } else { - false + None } }; - if !next { + let Some((start, before)) = next else { if closed.load(std::sync::atomic::Ordering::Relaxed) { break; } self.changed.notified().await; continue; - } - let start = buffers - .lock() - .unwrap() - .front() - .unwrap() - .oldest() - .expect("a resumed group has a frame"); - let before = revision.load(std::sync::atomic::Ordering::Relaxed); + }; let mapping = role.resume(start, None).await?; + let buffers = buffers.lock().unwrap(); if revision.load(std::sync::atomic::Ordering::Relaxed) != before { advancing.store(false, std::sync::atomic::Ordering::Relaxed); continue; } role.apply(mapping); - let buffers = buffers.lock().unwrap(); if let Some(end) = buffers.front().unwrap().end { role.end(end)?; } From 8dcdd7eff8321eacd3f0ea1b7635d535bd4b1a38 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 12:25:29 -0700 Subject: [PATCH 11/22] fix(play): wait for a decodable resumed video buffer Eviction past max_delay can empty a queued replacement until its next keyframe, and the resume step panicked on it. Advance only to a buffer with a frame, and discard an emptied one a later replacement follows. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/video.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 406563f5ed9..61859f48480 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -258,13 +258,18 @@ impl Video { // bumps the revision while holding it. let next = { let mut buffers = buffers.lock().unwrap(); - if buffers.len() > 1 { - advancing.store(true, std::sync::atomic::Ordering::Relaxed); - buffers.pop_front(); - let start = buffers.front().unwrap().oldest().expect("a resumed group has a frame"); - Some((start, revision.load(std::sync::atomic::Ordering::Relaxed))) - } else { - None + // Eviction can empty a replacement until its next keyframe; one a + // later replacement already follows never gets another. + while buffers.len() > 2 && buffers[1].oldest().is_none() { + buffers.remove(1); + } + match buffers.get(1).and_then(|next| next.oldest()) { + Some(start) => { + advancing.store(true, std::sync::atomic::Ordering::Relaxed); + buffers.pop_front(); + Some((start, revision.load(std::sync::atomic::Ordering::Relaxed))) + } + None => None, } }; let Some((start, before)) = next else { From 88bf2be40b15a20993bd2b7720de812194161bd7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 13:20:54 -0700 Subject: [PATCH 12/22] fix(play)!: key resumes by transition and report audio discontinuities Resume plans, paired mappings and queued video segments matched by queue position, so a skipped or doubled restart shifted every later pairing. Each declared end now opens or joins a shared transition that carries both roles' plans and the mapping, and a role that will never resume one abandons it. `moq_audio::decode::Read` drops `Gap` and renames `segment` to `discontinuity`: a bump with no `End` before it is a loss, which the next frame's timestamp already bounds. Also restores the `end` field in the play buffer's `Default` after the main merge. Co-Authored-By: Claude Opus 5.5 --- doc/setup/upgrade.md | 9 +- rs/moq-audio/src/decode/consumer.rs | 63 +++-- rs/moq-audio/src/encode/producer.rs | 2 +- rs/moq-audio/tests/import.rs | 2 +- rs/moq-audio/tests/roundtrip.rs | 2 +- rs/moq-c/src/audio.rs | 2 +- rs/moq-cli/src/play/buffer.rs | 1 + rs/moq-cli/src/play/media.rs | 278 ++++++++++++++++---- rs/moq-cli/src/play/timeline.rs | 395 ++++++++++++++++++++-------- rs/moq-cli/src/play/video.rs | 66 +++-- rs/moq-ffi/src/audio.rs | 2 +- 11 files changed, 613 insertions(+), 209 deletions(-) diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index 81e87e6619f..e16bc720db8 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -24,10 +24,11 @@ These land with the next breaking release, not the 2026-09-23 train. `catalog::hang::Consumer` with `.into()`, then read it through `catalog::Stream`. Its `Hang`, `Msf`, and `Closed` variants and `catalog::msf::Consumer::track` are private. -- **Rust audio reads carry track boundaries.** `moq_audio::decode::Consumer::read()` - returns `decode::Read` events. Handle `Frame { segment, frame }`, `Gap`, and - `End` separately; `None` still means the track is finished. An `End` can be - followed by final PCM in the same decoder segment before a fresh segment arrives. +- **Rust audio reads carry encoder ends.** `moq_audio::decode::Consumer::read()` + returns `decode::Read` events: `Frame { discontinuity, frame }` and + `End { discontinuity, end }`; `None` still means the track is finished. An + `End` can be followed by final PCM with the same `discontinuity` before a fresh + encoder bumps it. A bump with no `End` before it is a loss. Binding audio frame APIs keep their existing shape. - **JS catalog ranking names its media kind.** Replace `Catalog.ranked(renditions)` with `Catalog.Video.ranked(renditions)`. `Catalog.Audio.ranked(renditions)` diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 1711a3c0b14..eb40ec5077a 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -12,27 +12,25 @@ use crate::jitter::{self, Target}; use crate::resample::{Remix, Resampler}; use crate::{Activity, Error, Format, Frame, Layout}; -/// Decoded PCM and track boundaries in delivery order. +/// Decoded PCM and encoder ends in delivery order. +/// +/// `discontinuity` goes up wherever the decoded timeline breaks: at the fresh +/// encoder after an [`End`](Self::End), or at a loss. A bump with no `End` +/// before it is a loss, and the hole runs from the previous frame's end to the +/// new frame's timestamp. #[derive(Clone, Debug)] pub enum Read { - /// PCM from one decoder segment, including its drained tail. + /// Decoded PCM, including the drained tail before a discontinuity. Frame { - /// The decoder segment; only equality matters. - segment: u64, + /// The decoded timeline these samples belong to; only changes matter. + discontinuity: u64, /// The decoded samples and their presentation time. frame: Frame, }, - /// Media was lost before the next frame. - Gap { - /// The furthest presentation point read before the loss. - timestamp: moq_net::Timestamp, - /// The upper bound on the missing media span. - duration: moq_net::Timestamp, - }, - /// The publisher stopped this encoder; final PCM may still follow in its segment. + /// The publisher stopped this encoder; its final PCM may still follow. End { - /// The decoder segment containing the stopped encoder's final samples. - segment: u64, + /// The decoded timeline holding the stopped encoder's final samples. + discontinuity: u64, /// Samples at or past this point are padding and are trimmed. end: moq_net::Timestamp, }, @@ -118,8 +116,8 @@ pub struct Consumer { /// Frames decoded and not yet handed back, so a gap's tail can be returned /// ahead of the packet that exposed it. ready: VecDeque, - /// Stable across all groups and the resampler tail of one decoder segment. - segment: u64, + /// Stable across all groups and the resampler tail of one decoded timeline. + discontinuity: u64, /// Codec activity spans the resampler's buffered output still covers. spans: VecDeque, /// Activity of the last span the output ran past, for the rounding samples the @@ -236,7 +234,7 @@ impl Consumer { resolved_layout: layout, next_start: None, ready: VecDeque::new(), - segment: 0, + discontinuity: 0, spans: VecDeque::new(), trailing: Activity::Active, epoch: None, @@ -310,15 +308,15 @@ impl Consumer { // own group carries the stopped encoder's final packets, trimmed below. if self.end.is_some() && group != self.end_group { self.restart()?; - self.segment += 1; + self.discontinuity += 1; } frame } - Some(Encoded::Gap { timestamp, duration }) => { + // The loss shows as the next frame's discontinuity, with no end before it. + Some(Encoded::Gap { .. }) => { self.restart()?; - self.segment += 1; + self.discontinuity += 1; self.warmup_pending = !self.warmup.is_zero(); - self.ready.push_back(Read::Gap { timestamp, duration }); continue; } Some(Encoded::End { group, end }) => { @@ -327,13 +325,13 @@ impl Consumer { self.frames_decoded = 0; self.terminal_start = None; return Ok(Some(Read::End { - segment: self.segment, + discontinuity: self.discontinuity, end, })); } None => { - let segment = self.segment; - return Ok(self.flush()?.map(|frame| Read::Frame { segment, frame })); + let discontinuity = self.discontinuity; + return Ok(self.flush()?.map(|frame| Read::Frame { discontinuity, frame })); } }; @@ -363,7 +361,7 @@ impl Consumer { && let Some(frame) = self.gap()? { self.ready.push_back(Read::Frame { - segment: self.segment, + discontinuity: self.discontinuity, frame, }); } @@ -468,7 +466,7 @@ impl Consumer { // same iteration still comes out first. The next turn of the loop pops it. if let Some(frame) = self.frame(pcm, timestamp, activity)? { self.ready.push_back(Read::Frame { - segment: self.segment, + discontinuity: self.discontinuity, frame, }); } @@ -483,7 +481,7 @@ impl Consumer { fn restart(&mut self) -> Result<(), Error> { if let Some(frame) = self.drain()? { self.ready.push_back(Read::Frame { - segment: self.segment, + discontinuity: self.discontinuity, frame, }); } @@ -698,7 +696,7 @@ mod tests { loop { match consumer.read().await? { Some(Read::Frame { frame, .. }) => return Ok(Some(frame)), - Some(Read::Gap { .. } | Read::End { .. }) => {} + Some(Read::End { .. }) => {} None => return Ok(None), } } @@ -1072,13 +1070,12 @@ mod tests { let mut resumed = false; while let Some(event) = consumer.read().await.unwrap() { match event { - Read::End { segment, .. } if ended.is_none() => ended = Some(segment), - Read::Frame { segment, .. } if ended == Some(segment) => { + Read::End { discontinuity, .. } if ended.is_none() => ended = Some(discontinuity), + Read::Frame { discontinuity, .. } if ended == Some(discontinuity) => { assert!(!resumed); tail = true; } - Read::Frame { segment, .. } if ended.is_some_and(|old| old != segment) => resumed = true, - Read::Gap { .. } => panic!("a declared encoder boundary is not loss"), + Read::Frame { discontinuity, .. } if ended.is_some_and(|old| old != discontinuity) => resumed = true, _ => {} } } @@ -1086,7 +1083,7 @@ mod tests { tail, "the old resampler tail must follow its End without looking like a new encoder" ); - assert!(resumed, "a later group opens a fresh decoded segment"); + assert!(resumed, "a later group opens a fresh decoded timeline"); } /// A packet whose sample count isn't a multiple of the resampler's chunk leaves diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index ffc9322de68..4f3650f31c3 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -563,7 +563,7 @@ mod tests { loop { match consumer.read().await? { Some(crate::decode::Read::Frame { frame, .. }) => return Ok(Some(frame)), - Some(crate::decode::Read::Gap { .. } | crate::decode::Read::End { .. }) => {} + Some(crate::decode::Read::End { .. }) => {} None => return Ok(None), } } diff --git a/rs/moq-audio/tests/import.rs b/rs/moq-audio/tests/import.rs index 0e1faa69cb7..511d3702de2 100644 --- a/rs/moq-audio/tests/import.rs +++ b/rs/moq-audio/tests/import.rs @@ -67,7 +67,7 @@ async fn next_frame(consumer: &mut decode::Consumer) -> Result return Ok(Some(frame)), - Some(decode::Read::Gap { .. } | decode::Read::End { .. }) => {} + Some(decode::Read::End { .. }) => {} None => return Ok(None), } } diff --git a/rs/moq-audio/tests/roundtrip.rs b/rs/moq-audio/tests/roundtrip.rs index 68c64c820e3..3304c4b3e5a 100644 --- a/rs/moq-audio/tests/roundtrip.rs +++ b/rs/moq-audio/tests/roundtrip.rs @@ -276,7 +276,7 @@ async fn next_frame(consumer: &mut decode::Consumer) -> Result return Ok(Some(frame)), - Some(decode::Read::Gap { .. } | decode::Read::End { .. }) => {} + Some(decode::Read::End { .. }) => {} None => return Ok(None), } } diff --git a/rs/moq-c/src/audio.rs b/rs/moq-c/src/audio.rs index a7260f01770..58b3488ad43 100644 --- a/rs/moq-c/src/audio.rs +++ b/rs/moq-c/src/audio.rs @@ -260,7 +260,7 @@ impl Audio { _ = &mut close => return Ok(()), frame = consumer.read() => match frame { Ok(Some(moq_audio::decode::Read::Frame { frame, .. })) => frame, - Ok(Some(moq_audio::decode::Read::Gap { .. } | moq_audio::decode::Read::End { .. })) => continue, + Ok(Some(moq_audio::decode::Read::End { .. })) => continue, Ok(None) => return Ok(()), // One packet the codec rejected is that packet's problem: the // decoder stays usable, so drop it and keep the subscription rather diff --git a/rs/moq-cli/src/play/buffer.rs b/rs/moq-cli/src/play/buffer.rs index 746f70d1d47..66a92e52177 100644 --- a/rs/moq-cli/src/play/buffer.rs +++ b/rs/moq-cli/src/play/buffer.rs @@ -34,6 +34,7 @@ impl Default for Buffer { // A join resumes at its first keyframe, like any other break. waiting_keyframe: true, ended: false, + end: None, } } } diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 3158b740892..a7c1c3ec1fe 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -16,10 +16,18 @@ use tokio::time::Instant; use super::args::Args; use super::output::{Output, Sink, Speaker}; use super::playback::{Kind, Playback, joined}; -use super::timeline::{AudioTimeline, Presentation, Role, fit, timestamp}; +use super::timeline::{AudioTimeline, Presentation, Role, Transition, fit, timestamp}; use super::video::Video; use super::window::Event; +/// What a frame that starts a new decoded timeline crossed. +enum Crossing { + /// A declared end, whose transition the frame resumes. + Resume(Transition), + /// Lost media: no end was declared. + Loss, +} + /// The floor on the speaker's buffer, whatever delay was asked for. The device /// pulls on a fixed clock, so a ring shallower than this drops out on ordinary /// network jitter, and one with no depth at all can never be read from. @@ -455,11 +463,13 @@ async fn play_audio( // all logs once rather than once per packet. let mut dropping = false; - // A decoder segment only changes at a declared end or a gap, and a gap - // resets this, so any change of segment is a resume. A short segment can - // declare its end before its first PCM, so the end alone cannot mark it. + // Resolved in delivery order, ahead of playback, so each end's transition is + // declared before the replacement media that resumes it is planned. A short + // timeline can declare its end before its first PCM, so ends queue until a + // frame crosses them. let receive = async { - let mut planned = None; + let mut current = None; + let mut ends = VecDeque::new(); loop { let event = match consumer.read().await { Err(err) if !matches!(err, moq_audio::Error::Decode(_)) => return Err(err.into()), @@ -467,17 +477,38 @@ async fn play_audio( }; let ended = matches!(event, Ok(None)); let delay = consumer.delay(); - match &event { - Ok(Some(moq_audio::decode::Read::Gap { .. })) => planned = None, - Ok(Some(moq_audio::decode::Read::Frame { segment, frame })) => { - let resumed = planned.replace(*segment).is_some_and(|planned| planned != *segment); - if resumed { - role.plan(frame.timestamp, Some(lead(delay, frame))); + let crossing = match &event { + Ok(Some(moq_audio::decode::Read::End { discontinuity, .. })) => { + ends.push_back((*discontinuity, role.declare())); + None + } + Ok(Some(moq_audio::decode::Read::Frame { discontinuity, frame })) + if current + .replace(*discontinuity) + .is_some_and(|current| current != *discontinuity) => + { + // Every end before this timeline was crossed. Only the last one has + // replacement media; the ones before it ended empty timelines. + let mut crossed = None; + while let Some(&(ended, transition)) = ends.front() + && ended < *discontinuity + { + ends.pop_front(); + if let Some(skipped) = crossed.replace(transition) { + role.abandon(skipped); + } } + Some(match crossed { + Some(transition) => { + role.plan(transition, frame.timestamp, Some(lead(delay, frame))); + Crossing::Resume(transition) + } + None => Crossing::Loss, + }) } - _ => {} - } - received.lock().unwrap().push_back((event, delay)); + _ => None, + }; + received.lock().unwrap().push_back((event, delay, crossing)); arrived.notify_one(); if ended { return Ok::<_, anyhow::Error>(()); @@ -485,60 +516,63 @@ async fn play_audio( } }; let play = async { - // The segment the role's mapping serves, and an end declared for a later - // one, which can only be mapped once that segment resumes. + // The timeline the role's mapping serves, and an end declared for a later + // one, which can only be mapped once that timeline resumes. let mut mapped = None; let mut deferred = None; let mut sequential = false; loop { - let (event, delay) = loop { + let (event, delay, crossing) = loop { if let Some(event) = received.lock().unwrap().pop_front() { break event; } arrived.notified().await; }; let frame = match event { - Ok(Some(moq_audio::decode::Read::Frame { segment, frame })) => { - if mapped.replace(segment).is_some_and(|mapped| mapped != segment) { - // A declared boundary preserves the old sink, unlike an undeclared hole. - drain(sink).await; - let first = lead(delay, &frame); - role.apply(role.resume(frame.timestamp, Some(first)).await?); - let at = presentation - .lock() - .unwrap() - .due(role.map(frame.timestamp)?) - .and_then(|at| at.checked_sub(first)); - if let Some(at) = at { - tokio::time::sleep_until(at.into()).await; + Ok(Some(moq_audio::decode::Read::Frame { discontinuity, frame })) => { + mapped = Some(discontinuity); + match crossing { + Some(Crossing::Loss) => { + deferred = None; + role.gap(); + sequential = false; } - if let Some((ended, end)) = deferred.take() - && ended == segment - { - role.end(end)?; + Some(Crossing::Resume(transition)) => { + // A declared boundary preserves the old sink, unlike an undeclared hole. + drain(sink).await; + let first = lead(delay, &frame); + let mapping = role.resume(transition, frame.timestamp, Some(first)).await?; + role.apply(transition, mapping); + let at = presentation + .lock() + .unwrap() + .due(role.map(frame.timestamp)?) + .and_then(|at| at.checked_sub(first)); + if let Some(at) = at { + tokio::time::sleep_until(at.into()).await; + } + if let Some((ended, end)) = deferred.take() + && ended == discontinuity + { + role.end(end)?; + } + sequential = true; + timeline = AudioTimeline::default(); + sink = speaker.sink(input.clone())?; + dry = true; } - sequential = true; - timeline = AudioTimeline::default(); - sink = speaker.sink(input.clone())?; - dry = true; + None => {} } frame } - Ok(Some(moq_audio::decode::Read::End { segment, end })) => { - if mapped.is_none_or(|mapped| mapped == segment) { + Ok(Some(moq_audio::decode::Read::End { discontinuity, end })) => { + if mapped.is_none_or(|mapped| mapped == discontinuity) { role.end(end)?; } else { - deferred = Some((segment, end)); + deferred = Some((discontinuity, end)); } continue; } - Ok(Some(moq_audio::decode::Read::Gap { .. })) => { - mapped = None; - deferred = None; - role.gap(); - sequential = false; - continue; - } Ok(None) => break, // One bad packet is that packet's problem: the decoder stays usable, so // skip it rather than ending playback and taking the video window down @@ -905,6 +939,154 @@ mod tests { ); } + #[tokio::test] + async fn an_end_before_its_first_pcm_still_resumes_that_timeline() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Audio)); + let mut options = moq_audio::decode::Options::new(); + options.delay = Some(delay); + options.max_delay = delay; + options.output.format = moq_audio::Format::F32; + options.output.sample_rate = Some(SAMPLE_RATE); + let consumer = moq_audio::decode::Consumer::new( + &broadcast.consume(), + &AudioConfig::new(AudioCodec::Pcm, 44_100, 1), + "audio", + options, + ) + .await + .unwrap(); + let recorder = Recorder::default(); + let playback = AudioPlayback { + changed: Default::default(), + speaker: recorder.clone(), + presentation: Arc::new(Mutex::new(Presentation::new(delay))), + latency: delay, + output: recorder.clone(), + }; + let task = tokio::spawn(async move { + play_audio(consumer, playback).await.unwrap().finish().await; + }); + let pcm = |micros: u64, samples: usize, sample: f32| moq_mux::container::Frame { + timestamp: moq_net::Timestamp::from_micros(micros).unwrap(), + duration: Some(moq_net::Timestamp::from_scale(samples as u64, 44_100).unwrap()), + keyframe: true, + payload: std::iter::repeat_n(sample.to_le_bytes(), samples) + .flatten() + .collect::>() + .into(), + }; + for index in 0..4 { + producer.write(pcm(index * 23_220, 1024, 0.25)).unwrap(); + } + producer.discontinuity().unwrap(); + // Too short for the resampler to emit anything before this encoder ends too. + producer.write(pcm(100_000, 64, 0.5)).unwrap(); + producer.discontinuity().unwrap(); + for index in 0..4 { + producer.write(pcm(200_000 + index * 23_220, 1024, 0.75)).unwrap(); + } + producer.finish().unwrap(); + task.await.unwrap(); + let played = recorder.played(); + let sinks = |range: std::ops::Range| { + played + .iter() + .filter(|part| range.contains(&part.sample)) + .map(|part| part.sink) + .collect::>() + }; + let (old, short, new) = (sinks(0.2..0.3), sinks(0.4..0.6), sinks(0.7..0.8)); + assert!(!short.is_empty(), "the short timeline played: {played:?}"); + assert!( + old.is_disjoint(&short), + "the short timeline resumed on its own sink: {played:?}" + ); + assert!( + short.is_disjoint(&new), + "the next timeline resumed on its own sink: {played:?}" + ); + } + + #[tokio::test] + async fn an_evicted_replacement_gop_gives_way_to_the_next_replacement() { + struct Passthrough; + impl super::super::video::Decoder for Passthrough { + async fn decode(&mut self, frame: moq_mux::container::Frame) -> anyhow::Result> { + Ok(vec![moq_video::Frame::new( + moq_video::Surface::rgba(&[128; 16 * 16 * 4], moq_video::Size::new(16, 16))?, + frame.timestamp, + )]) + } + async fn flush(&mut self) -> anyhow::Result> { + Ok(Vec::new()) + } + } + tokio::time::pause(); + let delay = Duration::from_millis(200); + let recorder = Recorder::default(); + let origin = moq_tokio::origin::spawn(); + let media = media(&origin, delay, recorder.clone()); + let broadcast = moq_net::broadcast::Info::new().produce(); + let video = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + // The network keeps every group; only the player's age budget evicts. + let subscriber = video + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))) + .await + .unwrap(); + let mut video = moq_mux::container::Producer::new(video, Container::Legacy(moq_mux::container::Kind::Video)); + let video_play = Video { + presentation: media.presentation.clone(), + frames: media.video.clone(), + changed: media.drained.clone(), + output: recorder.clone(), + max_delay: delay, + }; + let video_task = tokio::spawn(video_play.run( + moq_mux::container::Consumer::new(subscriber, Container::Legacy(moq_mux::container::Kind::Video)), + Passthrough, + )); + let picture = |millis: u64, keyframe: bool| moq_mux::container::Frame { + timestamp: moq_net::Timestamp::from_millis(millis).unwrap(), + duration: Some(moq_net::Timestamp::from_millis(20).unwrap()), + keyframe, + payload: Bytes::from_static(b"picture"), + }; + for index in 0..3 { + video.write(picture(index * 20, index == 0)).unwrap(); + tokio::time::sleep(PACKET_DURATION).await; + } + video.discontinuity().unwrap(); + // A burst longer than the age budget evicts this replacement's only keyframe. + for index in 0..15 { + video.write(picture(index * 20, index == 0)).unwrap(); + } + video.discontinuity().unwrap(); + for index in 0..3 { + video.write(picture(5_000 + index * 20, index == 0)).unwrap(); + } + video.finish().unwrap(); + let mut shown = Vec::new(); + for _ in 0..2_000 { + tokio::task::yield_now().await; + if let Some(timestamp) = recorder.present(&media) { + shown.push(timestamp.as_millis()); + } + tokio::time::advance(Duration::from_millis(1)).await; + } + video_task.await.unwrap().unwrap(); + assert_eq!(shown[..3], [0, 20, 40], "{shown:?}"); + assert_eq!(shown.len(), 6, "the next replacement plays: {shown:?}"); + } + /// A mono PCM rendition, published and named in the catalog until dropped. fn rendition( broadcast: &moq_net::broadcast::Producer, diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index a1a27d406a9..b215878864f 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -31,19 +31,28 @@ pub(super) struct Presentation { /// will never play. restart: bool, /// Endpoints declared by each role, on the shared playback timeline. - roles: [Segment; 2], + ends: [Option; 2], + /// Encoder restarts whose replacements have not all started playing. + transitions: std::collections::VecDeque, + /// The latest transition each role declared. + declared: [Transition; 2], + next: Transition, reference: Option, } -#[derive(Default)] -struct Segment { - end: Option, - /// Whether a declared end is still waiting for its replacement. - pending: bool, - /// The peer's replacement mapping, applied while this role was pending. - paired: Option, - plans: std::collections::VecDeque, - active: Option, +/// One encoder restart, shared by the roles whose ends declared it. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +pub(super) struct Transition(u64); + +struct Pending { + id: Transition, + /// Roles whose declared end opened or joined this transition. + members: [bool; 2], + /// Members that applied its mapping. + resumed: [bool; 2], + /// Each member's first replacement media. + plans: [Option; 2], + mapping: Option, } struct Boundary { @@ -51,10 +60,10 @@ struct Boundary { end: Duration, } +#[derive(Clone, Copy)] struct Plan { source: Duration, delay: Duration, - mapping: Option, } impl Presentation { @@ -64,11 +73,29 @@ impl Presentation { delay, speaker: false, restart: false, - roles: Default::default(), + ends: Default::default(), + transitions: Default::default(), + declared: Default::default(), + next: Transition(1), reference: None, } } + fn transition(&mut self, id: Transition) -> Option<&mut Pending> { + self.transitions.iter_mut().find(|pending| pending.id == id) + } + + /// Forget transitions nobody resumes, or that every member resumed once + /// their replacement started and nothing can join them. + fn prune(&mut self, now: Instant) { + self.transitions.retain(|pending| { + let members = pending.members.iter().any(|member| *member); + let resumed = (0..2).all(|role| !pending.members[role] || pending.resumed[role]); + let started = pending.mapping.is_some_and(|mapping| mapping.started(now)); + members && !(resumed && started) + }); + } + /// The media position the shared clock will reach at `at`. fn position(&self, at: Instant) -> Option { let reference = self.reference?; @@ -195,6 +222,10 @@ impl Mapping { }; Ok(Timestamp::from_micros(u64::try_from(target.as_micros())?)?) } + + fn started(&self, now: Instant) -> bool { + self.starts.is_none_or(|starts| starts <= now) + } } impl Role { @@ -215,63 +246,122 @@ impl Role { self.mapping.lock().unwrap().map(source) } + /// Declare that this role's encoder stopped, returning the transition its + /// replacement resumes. + /// + /// A peer's transition this role has not joined yet is the same restart, as + /// long as its replacement has not started playing; otherwise this one opens + /// a transition of its own. + pub(super) fn declare(&self) -> Transition { + let now = tokio::time::Instant::now().into_std(); + let own = self.role as usize; + let mut presentation = self.presentation.lock().unwrap(); + presentation.prune(now); + let after = presentation.declared[own]; + let joined = presentation + .transitions + .iter_mut() + .filter(|transition| transition.id > after) + .find(|transition| { + transition.members[1 - own] + && !transition.members[own] + && transition.mapping.is_none_or(|mapping| !mapping.started(now)) + }) + .map(|transition| { + transition.members[own] = true; + transition.id + }); + let id = joined.unwrap_or_else(|| { + let id = presentation.next; + presentation.next = Transition(id.0 + 1); + let mut members = [false; 2]; + members[own] = true; + presentation.transitions.push_back(Pending { + id, + members, + resumed: [false; 2], + plans: [None, None], + mapping: None, + }); + id + }); + presentation.declared[own] = id; + drop(presentation); + self.changed.notify_waiters(); + id + } + + /// Mark where this role's current segment ends on the shared clock. pub(super) fn end(&self, source: Timestamp) -> anyhow::Result<()> { let mapping = *self.mapping.lock().unwrap(); - let mut presentation = self.presentation.lock().unwrap(); - let segment = &mut presentation.roles[self.role as usize]; - segment.end = Some(Boundary { + self.presentation.lock().unwrap().ends[self.role as usize] = Some(Boundary { starts: mapping.starts, end: timestamp(mapping.map(source)?), }); - segment.pending = true; - drop(presentation); self.changed.notify_waiters(); Ok(()) } - pub(super) fn plan(&self, source: Timestamp, delay: Option) { + /// Record the first replacement media for `transition`. + pub(super) fn plan(&self, transition: Transition, source: Timestamp, delay: Option) { let mut presentation = self.presentation.lock().unwrap(); let delay = delay.unwrap_or(presentation.delay); - presentation.roles[self.role as usize].plans.push_back(Plan { - source: Duration::from(source), - delay, - mapping: None, - }); + if let Some(pending) = presentation.transition(transition) { + pending.plans[self.role as usize] = Some(Plan { + source: Duration::from(source), + delay, + }); + } + drop(presentation); self.changed.notify_waiters(); } + /// Media was lost: nothing waits for the discarded tail. pub(super) fn gap(&self) { - self.presentation.lock().unwrap().roles[self.role as usize] = Segment::default(); + self.presentation.lock().unwrap().ends[self.role as usize] = None; + self.changed.notify_waiters(); + } + + /// This role will never resume `transition`, so a peer must not pair with its plan. + pub(super) fn abandon(&self, transition: Transition) { + let own = self.role as usize; + let mut presentation = self.presentation.lock().unwrap(); + if let Some(pending) = presentation.transition(transition) { + pending.members[own] = false; + pending.plans[own] = None; + } + presentation.prune(tokio::time::Instant::now().into_std()); + drop(presentation); self.changed.notify_waiters(); } - pub(super) fn apply(&self, mapping: Mapping) { + pub(super) fn apply(&self, transition: Transition, mapping: Mapping) { *self.mapping.lock().unwrap() = mapping; let mut presentation = self.presentation.lock().unwrap(); - let own = &mut presentation.roles[self.role as usize]; - own.active = Some(mapping); - own.plans.pop_front(); - own.pending = false; - own.paired = None; - // A peer that ended too resumes on this mapping, however late its media lands. - let peer = &mut presentation.roles[1 - self.role as usize]; - if peer.pending { - peer.paired = Some(mapping); + if let Some(pending) = presentation.transition(transition) { + pending.resumed[self.role as usize] = true; } + presentation.prune(tokio::time::Instant::now().into_std()); drop(presentation); self.changed.notify_waiters(); } - /// Drain declared old media, then align the new encoder to the running clock. - pub(super) async fn resume(&self, source: Timestamp, delay: Option) -> anyhow::Result { + /// Drain declared old media, then map `transition`'s replacement onto the + /// running clock. + pub(super) async fn resume( + &self, + transition: Transition, + source: Timestamp, + delay: Option, + ) -> anyhow::Result { loop { let now = tokio::time::Instant::now().into_std(); let end = { let presentation = self.presentation.lock().unwrap(); presentation - .roles + .ends .iter() - .filter_map(|role| role.end.as_ref()) + .flatten() .filter(|boundary| boundary.starts.is_none_or(|start| start <= now)) .filter_map(|boundary| presentation.due(media(boundary.end))) .max() @@ -284,69 +374,56 @@ impl Role { continue; } let mut presentation = self.presentation.lock().unwrap(); - let own = self.role as usize; - let peer = 1 - own; - // The peer already resumed the transition this role is finishing. - if let Some(mapping) = presentation.roles[own] - .plans - .front() - .and_then(|plan| plan.mapping) - .or(presentation.roles[own].paired) - { - return Ok(mapping); - } - // Both roles already have replacement media. The speaker's first sample - // defines their common origin even when its adaptive delay differs from video's. - if presentation.roles[own] - .plans - .front() - .is_some_and(|plan| plan.mapping.is_none()) - && presentation.roles[peer] - .plans - .front() - .is_some_and(|plan| plan.mapping.is_none()) - { - let audio = presentation.roles[Kind::Audio as usize].plans.front().unwrap(); - let starts = now - .checked_add(audio.delay) - .context("playback delay exceeds the system clock")?; - let mapping = Mapping { - source: audio.source, - target: presentation.position(starts).unwrap_or(audio.source), - starts: Some(starts), - }; - presentation.roles[own].plans.front_mut().unwrap().mapping = Some(mapping); - presentation.roles[peer].plans.front_mut().unwrap().mapping = Some(mapping); - return Ok(mapping); - } - if let Some(mapping) = presentation.roles[peer] - .active - .filter(|mapping| mapping.starts.is_some_and(|starts| starts > now)) - { + let peer = 1 - self.role as usize; + // A transition abandoned under this resume, by a gap the caller rechecks, + // maps alone like one the peer never joined. + let (mapping, paired) = presentation.transition(transition).map_or((None, None), |pending| { + let both = pending.members[peer] && pending.plans.iter().all(Option::is_some); + (pending.mapping, pending.plans[Kind::Audio as usize].filter(|_| both)) + }); + // The peer already resumed this transition. + if let Some(mapping) = mapping { return Ok(mapping); } + // Both roles have replacement media. The speaker's first sample defines + // their common origin even when its adaptive delay differs from video's. + // Otherwise this role maps alone and a peer joining later follows it. + let (source, delay) = match paired { + Some(audio) => (audio.source, audio.delay), + None => (Duration::from(source), delay.unwrap_or(presentation.delay)), + }; let starts = now - .checked_add(delay.unwrap_or(presentation.delay)) + .checked_add(delay) .context("playback delay exceeds the system clock")?; - return Ok(Mapping { - source: Duration::from(source), - target: presentation.position(starts).unwrap_or_else(|| timestamp(source)), + let mapping = Mapping { + source, + target: presentation.position(starts).unwrap_or(source), starts: Some(starts), - }); + }; + if let Some(pending) = presentation.transition(transition) { + pending.mapping = Some(mapping); + } + return Ok(mapping); } } } impl Drop for Role { fn drop(&mut self) { + let own = self.role as usize; if let Ok(mut presentation) = self.presentation.lock() { // A declared end stays: the sink handed back still plays the tail up to it, // and once it is past, nothing waits on it. - let segment = &mut presentation.roles[self.role as usize]; - *segment = Segment { - end: segment.end.take(), - ..Default::default() - }; + for pending in presentation + .transitions + .iter_mut() + .filter(|pending| !pending.resumed[own]) + { + pending.members[own] = false; + pending.plans[own] = None; + } + presentation.declared[own] = Transition::default(); + presentation.prune(tokio::time::Instant::now().into_std()); } self.changed.notify_waiters(); } @@ -475,10 +552,11 @@ mod tests { let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); let video = Role::new(presentation.clone(), Default::default(), Kind::Video); presentation.lock().unwrap().video(ms(10_000), start.into_std()); + let transition = audio.declare(); audio.end(ms(10_020)).unwrap(); tokio::time::advance(DELAY + Duration::from_millis(20)).await; let unchanged = presentation.lock().unwrap().due(ms(10_500)); - audio.apply(audio.resume(ms(0), Some(DELAY)).await.unwrap()); + audio.apply(transition, audio.resume(transition, ms(0), Some(DELAY)).await.unwrap()); assert_eq!(audio.map(ms(0)).unwrap().as_millis(), 10_120); assert_eq!(video.map(ms(10_500)).unwrap().as_millis(), 10_500); assert_eq!(presentation.lock().unwrap().due(ms(10_500)), unchanged); @@ -495,15 +573,17 @@ mod tests { .audio(Duration::from_secs(10), DELAY, now.into_std()); let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let transition = audio.declare(); + assert_eq!(video.declare(), transition, "one restart"); audio.end(ms(10_020)).unwrap(); video.end(ms(10_020)).unwrap(); - audio.plan(ms(0), Some(DELAY)); + audio.plan(transition, ms(0), Some(DELAY)); tokio::time::advance(DELAY + Duration::from_millis(20)).await; - audio.apply(audio.resume(ms(0), Some(DELAY)).await.unwrap()); + audio.apply(transition, audio.resume(transition, ms(0), Some(DELAY)).await.unwrap()); // The video replacement lands after the resumed audio has started playing. tokio::time::advance(DELAY * 2).await; - video.plan(ms(40), None); - video.apply(video.resume(ms(40), None).await.unwrap()); + video.plan(transition, ms(40), None); + video.apply(transition, video.resume(transition, ms(40), None).await.unwrap()); assert_eq!( video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), 40 @@ -521,22 +601,117 @@ mod tests { .audio(Duration::from_secs(10), DELAY, now.into_std()); let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let transition = audio.declare(); + video.declare(); audio.end(ms(10_020)).unwrap(); video.end(ms(10_020)).unwrap(); - video.plan(ms(40), None); + video.plan(transition, ms(40), None); tokio::time::advance(DELAY + Duration::from_millis(20)).await; - video.apply(video.resume(ms(40), None).await.unwrap()); + video.apply(transition, video.resume(transition, ms(40), None).await.unwrap()); // Video already queued its next replacement when audio's first one lands. + let next = video.declare(); + assert_ne!(next, transition); video.end(ms(60)).unwrap(); - video.plan(ms(5_000), None); - audio.plan(ms(0), Some(DELAY)); - audio.apply(audio.resume(ms(0), Some(DELAY)).await.unwrap()); + video.plan(next, ms(5_000), None); + audio.plan(transition, ms(0), Some(DELAY)); + audio.apply(transition, audio.resume(transition, ms(0), Some(DELAY)).await.unwrap()); assert_eq!( video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), 40 ); } + #[tokio::test] + async fn a_peer_two_restarts_ahead_pairs_with_the_first() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); + let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + video.plan(transition, ms(40), None); + let first = video.resume(transition, ms(40), None).await.unwrap(); + video.apply(transition, first); + // Video restarts again and resumes before audio's first replacement lands. + let next = video.declare(); + video.plan(next, ms(5_000), None); + tokio::time::advance(Duration::from_millis(10)).await; + video.apply(next, video.resume(next, ms(5_000), None).await.unwrap()); + audio.plan(transition, ms(0), Some(DELAY)); + audio.apply(transition, audio.resume(transition, ms(0), Some(DELAY)).await.unwrap()); + assert_eq!( + first.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + 40 + ); + } + + #[tokio::test] + async fn an_abandoned_replacement_leaves_the_next_transition_alone() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); + let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + // Video's first replacement is evicted before it plays, then it restarts again. + video.plan(transition, ms(40), None); + video.abandon(transition); + let next = video.declare(); + video.plan(next, ms(5_000), None); + audio.plan(transition, ms(0), Some(DELAY)); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + audio.apply(transition, audio.resume(transition, ms(0), Some(DELAY)).await.unwrap()); + video.apply(next, video.resume(next, ms(5_000), None).await.unwrap()); + // The later replacement maps onto the running clock, not the audio restart's. + let resumed = tokio::time::Instant::now() + DELAY; + assert_eq!( + presentation.lock().unwrap().due(video.map(ms(5_000)).unwrap()), + Some(resumed.into_std()) + ); + } + + #[tokio::test] + async fn an_abandoned_peer_plan_is_not_paired() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); + let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + audio.plan(transition, ms(0), Some(Duration::from_millis(80))); + audio.abandon(transition); + video.plan(transition, ms(40), None); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + video.apply(transition, video.resume(transition, ms(40), None).await.unwrap()); + let resumed = tokio::time::Instant::now() + DELAY; + assert_eq!( + presentation.lock().unwrap().due(video.map(ms(40)).unwrap()), + Some(resumed.into_std()) + ); + } + #[tokio::test] async fn a_closed_role_still_holds_its_declared_end() { tokio::time::pause(); @@ -548,12 +723,18 @@ mod tests { .audio(Duration::from_secs(10), DELAY, now.into_std()); let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + audio.declare(); audio.end(ms(10_200)).unwrap(); // The track closed, but its returned sink still plays the tail up to the end. drop(audio); + let transition = video.declare(); video.end(ms(10_000)).unwrap(); - video.plan(ms(0), None); - let resume = tokio::time::timeout(DELAY + Duration::from_millis(150), video.resume(ms(0), None)).await; + video.plan(transition, ms(0), None); + let resume = tokio::time::timeout( + DELAY + Duration::from_millis(150), + video.resume(transition, ms(0), None), + ) + .await; assert!(resume.is_err(), "video resumed before the audio tail played"); } @@ -568,13 +749,21 @@ mod tests { .audio(Duration::from_secs(10), DELAY, now.into_std()); let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let transition = audio.declare(); + video.declare(); audio.end(ms(10_020)).unwrap(); video.end(ms(10_020)).unwrap(); - audio.plan(ms(0), Some(Duration::from_millis(80))); - video.plan(ms(40), None); + audio.plan(transition, ms(0), Some(Duration::from_millis(80))); + video.plan(transition, ms(40), None); tokio::time::advance(DELAY + Duration::from_millis(20)).await; - video.apply(video.resume(ms(40), None).await.unwrap()); - audio.apply(audio.resume(ms(0), Some(Duration::from_millis(80))).await.unwrap()); + video.apply(transition, video.resume(transition, ms(40), None).await.unwrap()); + audio.apply( + transition, + audio + .resume(transition, ms(0), Some(Duration::from_millis(80))) + .await + .unwrap(), + ); assert_eq!( video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), 40 diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 721ec8d8837..709e79d6268 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -9,7 +9,7 @@ use tokio::time::Instant; use super::buffer::Buffer; use super::output::Output; -use super::timeline::{Presentation, Role}; +use super::timeline::{Presentation, Role, Transition}; use super::window::Event; /// A few surfaces, independent of the requested playout delay. A codec batch @@ -70,12 +70,34 @@ impl Decoder for moq_video::decode::Sink { } } +/// One encoder segment's media, and the transition it resumes when it +/// replaces the segment before it. +#[derive(Default)] +struct Segment { + media: Buffer, + resumes: Option, +} + +impl std::ops::Deref for Segment { + type Target = Buffer; + + fn deref(&self) -> &Buffer { + &self.media + } +} + +impl std::ops::DerefMut for Segment { + fn deref_mut(&mut self) -> &mut Buffer { + &mut self.media + } +} + impl Video { pub async fn run(self, mut track: Track, mut decoder: impl Decoder) -> anyhow::Result<()> { let closed = std::sync::atomic::AtomicBool::new(false); let advancing = std::sync::atomic::AtomicBool::new(false); let revision = std::sync::atomic::AtomicU64::new(0); - let buffers = Mutex::new(VecDeque::from([Buffer::default()])); + let buffers = Mutex::new(VecDeque::from([Segment::default()])); let role = Role::new( self.presentation.clone(), self.changed.clone(), @@ -83,7 +105,8 @@ impl Video { ); let receive = async { // The group of a declared end whose restart waits for a later group, so the - // media buffered before the end still plays out. + // media buffered before the end still plays out, and the transition that + // later group resumes. let mut end = None; loop { @@ -93,9 +116,12 @@ impl Video { let before = buffers.front().unwrap().generation; let ended = match read { Some(Read::Frame { group, frame }) => { - if end.take_if(|end| *end != group).is_some() { - role.plan(frame.timestamp, None); - buffers.push_back(Buffer::default()); + if let Some((_, transition)) = end.take_if(|(end, _)| *end != group) { + role.plan(transition, frame.timestamp, None); + buffers.push_back(Segment { + resumes: Some(transition), + ..Default::default() + }); } if active && buffers.len() == 1 && self @@ -111,18 +137,24 @@ impl Video { } // Applied in delivery order, so the restart lands before the frame after it. Some(Read::Gap { .. }) => { - end = None; + let queued = buffers.iter().filter_map(|buffer| buffer.resumes); + for transition in end.take().map(|(_, transition)| transition).into_iter().chain(queued) { + role.abandon(transition); + } revision.fetch_add(1, std::sync::atomic::Ordering::Relaxed); role.gap(); buffers.clear(); - buffers.push_back(Buffer::default()); + buffers.push_back(Segment::default()); buffers.front_mut().unwrap().clear(); buffers.front_mut().unwrap().generation = before + 1; self.presentation.lock().unwrap().video_restarted(); false } Some(Read::End { group, end: endpoint }) => { - end = Some(group); + if let Some((_, transition)) = end.take() { + role.abandon(transition); + } + end = Some((group, role.declare())); buffers.back_mut().unwrap().ended = true; buffers.back_mut().unwrap().end = Some(endpoint); if active { @@ -261,31 +293,33 @@ impl Video { // Eviction can empty a replacement until its next keyframe; one a // later replacement already follows never gets another. while buffers.len() > 2 && buffers[1].oldest().is_none() { - buffers.remove(1); + if let Some(transition) = buffers.remove(1).and_then(|skipped| skipped.resumes) { + role.abandon(transition); + } } - match buffers.get(1).and_then(|next| next.oldest()) { - Some(start) => { + match buffers.get(1).and_then(|next| Some((next.oldest()?, next.resumes?))) { + Some((start, transition)) => { advancing.store(true, std::sync::atomic::Ordering::Relaxed); buffers.pop_front(); - Some((start, revision.load(std::sync::atomic::Ordering::Relaxed))) + Some((start, transition, revision.load(std::sync::atomic::Ordering::Relaxed))) } None => None, } }; - let Some((start, before)) = next else { + let Some((start, transition, before)) = next else { if closed.load(std::sync::atomic::Ordering::Relaxed) { break; } self.changed.notified().await; continue; }; - let mapping = role.resume(start, None).await?; + let mapping = role.resume(transition, start, None).await?; let buffers = buffers.lock().unwrap(); if revision.load(std::sync::atomic::Ordering::Relaxed) != before { advancing.store(false, std::sync::atomic::Ordering::Relaxed); continue; } - role.apply(mapping); + role.apply(transition, mapping); if let Some(end) = buffers.front().unwrap().end { role.end(end)?; } diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index faf8ea07742..9626233258d 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -345,7 +345,7 @@ impl ConsumerInner { loop { match self.consumer.read().await? { Some(moq_audio::decode::Read::Frame { frame, .. }) => return Ok(Some(frame.into())), - Some(moq_audio::decode::Read::Gap { .. } | moq_audio::decode::Read::End { .. }) => {} + Some(moq_audio::decode::Read::End { .. }) => {} None => return Ok(None), } } From ef725c9d9ae69552ab350a0e7949d922cf6c8986 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 13:33:47 -0700 Subject: [PATCH 13/22] fix(audio): count an undeclared timestamp hole as a discontinuity A packet that no longer continued the decoded timeline reset the decoder and drained the resampler without bumping the counter, so callers could not see the loss. Bump it after the drained tail. Co-Authored-By: Claude Opus 5.5 --- rs/moq-audio/src/decode/consumer.rs | 52 ++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index eb40ec5077a..ccafa680887 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -358,12 +358,14 @@ impl Consumer { && self .next_start .is_some_and(|next| discontinuous(next, mux_frame.timestamp)) - && let Some(frame) = self.gap()? { - self.ready.push_back(Read::Frame { - discontinuity: self.discontinuity, - frame, - }); + if let Some(frame) = self.gap()? { + self.ready.push_back(Read::Frame { + discontinuity: self.discontinuity, + frame, + }); + } + self.discontinuity += 1; } let rate = self.decoder.sample_rate(); @@ -1024,6 +1026,46 @@ mod tests { } } + #[tokio::test] + async fn an_undeclared_timestamp_hole_is_a_discontinuity() { + tokio::time::pause(); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 48_000, 1); + let options = Options { + max_delay: Duration::from_secs(1), + ..Options::new() + }; + let mut consumer = Consumer::new(&broadcast.consume(), &catalog, "audio", options) + .await + .unwrap(); + // Contiguous groups, so the container sees no gap: only the stamps jump. + for micros in [0, 20_000, 500_000] { + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_micros(micros).unwrap(), + duration: Some(Timestamp::from_micros(20_000).unwrap()), + keyframe: true, + payload: (0..960).flat_map(|_| 0.25f32.to_le_bytes()).collect::>().into(), + }) + .unwrap(); + } + producer.finish().unwrap(); + let mut timelines = Vec::new(); + while let Some(event) = consumer.read().await.unwrap() { + if let Read::Frame { discontinuity, frame } = event { + timelines.push((frame.timestamp.as_micros(), discontinuity)); + } + } + assert_eq!(timelines, [(0, 0), (20_000, 0), (500_000, 1)]); + } + #[tokio::test] async fn decoded_segments_keep_resampler_tails_before_fresh_encoder_frames() { tokio::time::pause(); From 33099cd9039751038b36fff9560d7a098899b489 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 14:32:50 -0700 Subject: [PATCH 14/22] fix(play): resume the first frame after an end seen while joining A subscription that joined a paused track saw its end marker before any frame, so the first replacement frame never resumed that transition and left a stale endpoint for the peer to wait on. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/media.rs | 69 ++++++++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 5c4a1df7259..4666c160f32 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -497,10 +497,11 @@ async fn play_audio( None } Ok(Some(moq_audio::decode::Read::Frame { discontinuity, frame })) - if current - .replace(*discontinuity) - .is_some_and(|current| current != *discontinuity) => + if current != Some(*discontinuity) => { + // The first frame continues nothing, unless a track joined while paused + // declared an end before it. + let first = current.replace(*discontinuity).is_none(); // Every end before this timeline was crossed. Only the last one has // replacement media; the ones before it ended empty timelines. let mut crossed = None; @@ -512,13 +513,14 @@ async fn play_audio( role.abandon(skipped); } } - Some(match crossed { + match crossed { Some(transition) => { role.plan(transition, frame.timestamp, Some(lead(delay, frame))); - Crossing::Resume(transition) + Some(Crossing::Resume(transition)) } - None => Crossing::Loss, - }) + None if first => None, + None => Some(Crossing::Loss), + } } _ => None, }; @@ -1027,6 +1029,59 @@ mod tests { ); } + #[tokio::test] + async fn joining_a_paused_track_resumes_on_its_first_frame() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Audio)); + for index in 0..3 { + producer.write(packet(index, 0.25)).unwrap(); + } + producer.discontinuity().unwrap(); + // Joins at the end marker: the stopped encoder's media is already behind it. + let mut options = moq_audio::decode::Options::new(); + options.delay = Some(delay); + options.max_delay = delay; + options.start = moq_audio::decode::Start::Latest; + options.output.format = moq_audio::Format::F32; + let consumer = moq_audio::decode::Consumer::new( + &broadcast.consume(), + &AudioConfig::new(AudioCodec::Pcm, SAMPLE_RATE, 1), + "audio", + options, + ) + .await + .unwrap(); + let recorder = Recorder::default(); + let playback = AudioPlayback { + changed: Default::default(), + speaker: recorder.clone(), + presentation: Arc::new(Mutex::new(Presentation::new(delay))), + latency: delay, + output: recorder.clone(), + }; + let task = tokio::spawn(async move { + play_audio(consumer, playback).await.unwrap().finish().await; + }); + tokio::task::yield_now().await; + for index in 3..6 { + producer.write(packet(index, 0.5)).unwrap(); + } + producer.finish().unwrap(); + task.await.unwrap(); + let played = recorder.played(); + let resumed: Vec<_> = played.iter().filter(|part| part.sample == 0.5).collect(); + assert!(!resumed.is_empty(), "{played:?}"); + assert!( + resumed.iter().all(|part| part.sink > 0), + "the replacement resumed its declared transition: {played:?}" + ); + } + #[tokio::test] async fn an_evicted_replacement_gop_gives_way_to_the_next_replacement() { struct Passthrough; From 2a37432f2aeeaa126bf75c374ef79592bfbcc8f8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 14:39:16 -0700 Subject: [PATCH 15/22] fix(play): take a resumed transition's mapping without waiting A role joining a transition its peer already mapped waited for every boundary first, including ends the peer declared after resuming, so it missed its start. Return the retained mapping before waiting. Also drop a video flush's pictures when a gap replaced their generation during the await. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/timeline.rs | 36 ++++++++++++++++++++++++++++++++- rs/moq-cli/src/play/video.rs | 9 +++++++-- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index b215878864f..588cc898e9c 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -357,7 +357,12 @@ impl Role { loop { let now = tokio::time::Instant::now().into_std(); let end = { - let presentation = self.presentation.lock().unwrap(); + let mut presentation = self.presentation.lock().unwrap(); + // The peer already resumed this transition after draining every end + // before it; one it declared since must not hold this role back. + if let Some(mapping) = presentation.transition(transition).and_then(|pending| pending.mapping) { + return Ok(mapping); + } presentation .ends .iter() @@ -712,6 +717,35 @@ mod tests { ); } + #[tokio::test] + async fn a_resumed_transition_ignores_the_peers_later_end() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); + let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + video.plan(transition, ms(40), None); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + let mapping = video.resume(transition, ms(40), None).await.unwrap(); + video.apply(transition, mapping); + // The resumed video plays, then its encoder stops again far ahead. + tokio::time::advance(DELAY * 2).await; + video.declare(); + video.end(ms(60_000)).unwrap(); + audio.plan(transition, ms(0), Some(DELAY)); + let resumed = + tokio::time::timeout(Duration::from_millis(1), audio.resume(transition, ms(0), Some(DELAY))).await; + assert!(resumed.is_ok(), "audio waited for video's next end"); + } + #[tokio::test] async fn a_closed_role_still_holds_its_declared_end() { tokio::time::pause(); diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 709e79d6268..7965f3df250 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -332,9 +332,14 @@ impl Video { continue; } Step::Flush => { - generation = None; + let flushed = generation.take(); held.clear(); - decoder.flush().await? + let frames = decoder.flush().await?; + // A gap during the await already discarded these pictures' generation. + if flushed != Some(buffers.lock().unwrap().front().unwrap().generation) { + continue; + } + frames } }; // A codec returns display order, so a picture held before the newest From 313d9df703666b9843b7fcf671e5688daf12d29d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 14:48:21 -0700 Subject: [PATCH 16/22] fix(play): resume video across a loss and release cancelled ends A video gap while a replacement was queued abandoned it and kept decoding the replacement encoder on the old mapping. Like audio, an end declared before the loss still holds: the stand-in for the ended segment hands over to the latest queued transition. A role now keeps its declared end only when retired with a sink handed back for draining; a cancelled role releases it so the peer does not wait for a discarded tail. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/media.rs | 1 + rs/moq-cli/src/play/timeline.rs | 43 ++++++++++++++++++++++++++++++--- rs/moq-cli/src/play/video.rs | 20 ++++++++++++--- 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 4666c160f32..cbb831aed0d 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -679,6 +679,7 @@ async fn play_audio( Ok::<_, anyhow::Error>(sink) }; let (_, sink) = tokio::try_join!(receive, play)?; + role.retire(); Ok(sink) } diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index 588cc898e9c..964471b1ffb 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -199,6 +199,8 @@ pub(super) struct Role { changed: std::sync::Arc, role: Kind, mapping: std::sync::Mutex, + /// Whether a sink handed back for draining still plays up to the declared end. + retired: bool, } #[derive(Clone, Copy, Default)] @@ -239,9 +241,16 @@ impl Role { changed, role, mapping: Default::default(), + retired: false, } } + /// Stop, keeping the declared end: the sink handed back still plays the tail + /// up to it. Dropping instead discards the tail with the end. + pub(super) fn retire(mut self) { + self.retired = true; + } + pub(super) fn map(&self, source: Timestamp) -> anyhow::Result { self.mapping.lock().unwrap().map(source) } @@ -417,8 +426,10 @@ impl Drop for Role { fn drop(&mut self) { let own = self.role as usize; if let Ok(mut presentation) = self.presentation.lock() { - // A declared end stays: the sink handed back still plays the tail up to it, - // and once it is past, nothing waits on it. + // A retired end stays until it is past; nothing waits on it after that. + if !self.retired { + presentation.ends[own] = None; + } for pending in presentation .transitions .iter_mut() @@ -746,6 +757,32 @@ mod tests { assert!(resumed.is_ok(), "audio waited for video's next end"); } + #[tokio::test] + async fn a_cancelled_role_releases_its_declared_end() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); + let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + audio.declare(); + audio.end(ms(10_200)).unwrap(); + // Cancelled with its sink, so nothing plays the tail. + drop(audio); + let transition = video.declare(); + video.end(ms(10_000)).unwrap(); + video.plan(transition, ms(0), None); + let resume = tokio::time::timeout( + DELAY + Duration::from_millis(150), + video.resume(transition, ms(0), None), + ) + .await; + assert!(resume.is_ok(), "video waited for discarded audio"); + } + #[tokio::test] async fn a_closed_role_still_holds_its_declared_end() { tokio::time::pause(); @@ -760,7 +797,7 @@ mod tests { audio.declare(); audio.end(ms(10_200)).unwrap(); // The track closed, but its returned sink still plays the tail up to the end. - drop(audio); + audio.retire(); let transition = video.declare(); video.end(ms(10_000)).unwrap(); video.plan(transition, ms(0), None); diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 7965f3df250..13fd99de68a 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -136,17 +136,29 @@ impl Video { false } // Applied in delivery order, so the restart lands before the frame after it. + // An end declared before the loss still holds, as on the audio side: what + // follows comes from the replacement encoder and resumes the latest + // transition, while the media lost with the gap is discarded. Some(Read::Gap { .. }) => { - let queued = buffers.iter().filter_map(|buffer| buffer.resumes); - for transition in end.take().map(|(_, transition)| transition).into_iter().chain(queued) { + let mut queued: Vec<_> = buffers.iter().filter_map(|buffer| buffer.resumes).collect(); + let resumes = queued.pop(); + for transition in queued { role.abandon(transition); } revision.fetch_add(1, std::sync::atomic::Ordering::Relaxed); role.gap(); buffers.clear(); buffers.push_back(Segment::default()); - buffers.front_mut().unwrap().clear(); - buffers.front_mut().unwrap().generation = before + 1; + let front = buffers.front_mut().unwrap(); + front.clear(); + front.generation = before + 1; + front.ended = resumes.is_some() || end.is_some(); + if resumes.is_some() { + buffers.push_back(Segment { + resumes, + ..Default::default() + }); + } self.presentation.lock().unwrap().video_restarted(); false } From c30770b0b0bb2db402a270edfafab72e87da310e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 15:23:34 -0700 Subject: [PATCH 17/22] fix(play): freeze a retiring role's end on the wall clock A closed video track dropped its role while pictures were still queued, releasing its declared end early. A retired audio sink kept its end as a media timestamp, which a replacement rendition's re-pin could move far into the future. Both now hand the instant their tail finishes to Role::retire, which holds a declared end until then. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/media.rs | 2 +- rs/moq-cli/src/play/timeline.rs | 63 ++++++++++++++++++++++++++------- rs/moq-cli/src/play/video.rs | 60 +++++++++++++++++++++++++++---- 3 files changed, 105 insertions(+), 20 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index cbb831aed0d..aadcad16993 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -679,7 +679,7 @@ async fn play_audio( Ok::<_, anyhow::Error>(sink) }; let (_, sink) = tokio::try_join!(receive, play)?; - role.retire(); + role.retire(Instant::now().into_std().checked_add(sink.buffered())); Ok(sink) } diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index 964471b1ffb..565b35c5ffe 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -55,9 +55,11 @@ struct Pending { mapping: Option, } -struct Boundary { - starts: Option, - end: Duration, +enum Boundary { + /// A playing role's end on the shared timeline, which moves with the anchor. + Mapped { starts: Option, end: Duration }, + /// When a retired role's tail finishes playing, whatever the anchor does after. + Drained(Instant), } #[derive(Clone, Copy)] @@ -199,7 +201,7 @@ pub(super) struct Role { changed: std::sync::Arc, role: Kind, mapping: std::sync::Mutex, - /// Whether a sink handed back for draining still plays up to the declared end. + /// Whether the declared end was frozen for a tail that outlives the role. retired: bool, } @@ -245,9 +247,16 @@ impl Role { } } - /// Stop, keeping the declared end: the sink handed back still plays the tail - /// up to it. Dropping instead discards the tail with the end. - pub(super) fn retire(mut self) { + /// Stop while a tail still plays until `drained`, holding a declared end until + /// then. Dropping instead discards the tail with the end. + /// + /// The end is frozen on the wall clock: a later rendition re-pins the anchor, + /// which would carry a media timestamp onto an unrelated timeline. + pub(super) fn retire(mut self, drained: Option) { + let mut presentation = self.presentation.lock().unwrap(); + let end = &mut presentation.ends[self.role as usize]; + *end = end.take().and(drained.map(Boundary::Drained)); + drop(presentation); self.retired = true; } @@ -303,7 +312,7 @@ impl Role { /// Mark where this role's current segment ends on the shared clock. pub(super) fn end(&self, source: Timestamp) -> anyhow::Result<()> { let mapping = *self.mapping.lock().unwrap(); - self.presentation.lock().unwrap().ends[self.role as usize] = Some(Boundary { + self.presentation.lock().unwrap().ends[self.role as usize] = Some(Boundary::Mapped { starts: mapping.starts, end: timestamp(mapping.map(source)?), }); @@ -376,8 +385,13 @@ impl Role { .ends .iter() .flatten() - .filter(|boundary| boundary.starts.is_none_or(|start| start <= now)) - .filter_map(|boundary| presentation.due(media(boundary.end))) + .filter_map(|boundary| match boundary { + Boundary::Mapped { starts, end } => starts + .is_none_or(|start| start <= now) + .then(|| presentation.due(media(*end))) + .flatten(), + Boundary::Drained(at) => Some(*at), + }) .max() }; if let Some(end) = end.filter(|end| *end > now) { @@ -426,7 +440,7 @@ impl Drop for Role { fn drop(&mut self) { let own = self.role as usize; if let Ok(mut presentation) = self.presentation.lock() { - // A retired end stays until it is past; nothing waits on it after that. + // A retired end stays until it drains; nothing waits on it after that. if !self.retired { presentation.ends[own] = None; } @@ -797,7 +811,7 @@ mod tests { audio.declare(); audio.end(ms(10_200)).unwrap(); // The track closed, but its returned sink still plays the tail up to the end. - audio.retire(); + audio.retire(Some((now + Duration::from_millis(300)).into_std())); let transition = video.declare(); video.end(ms(10_000)).unwrap(); video.plan(transition, ms(0), None); @@ -809,6 +823,31 @@ mod tests { assert!(resume.is_err(), "video resumed before the audio tail played"); } + #[tokio::test] + async fn a_retired_end_ignores_a_later_re_pin() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); + let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + audio.declare(); + audio.end(ms(10_200)).unwrap(); + audio.retire(Some((now + Duration::from_millis(300)).into_std())); + // A replacement rendition starts over at zero and re-pins the anchor. + let mut pinned = presentation.lock().unwrap(); + pinned.restarted(); + pinned.audio(Duration::from_millis(50), DELAY, now.into_std()); + drop(pinned); + let transition = video.declare(); + video.plan(transition, ms(0), None); + let resume = tokio::time::timeout(Duration::from_millis(400), video.resume(transition, ms(0), None)).await; + assert!(resume.is_ok(), "video waited past the drained tail"); + } + #[tokio::test] async fn paired_plans_keep_media_offsets_and_follow_the_speakers_delay() { tokio::time::pause(); diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 13fd99de68a..351cfc58872 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -204,7 +204,7 @@ impl Video { let mut held = BTreeSet::new(); // Decoded pictures waiting for room in the window's queue. let mut pending = VecDeque::new(); - loop { + let drained = loop { let (current, oldest, ended) = { let buffers = buffers.lock().unwrap(); let buffer = buffers.front().unwrap(); @@ -255,9 +255,10 @@ impl Video { } else if generation.is_some() { Step::Flush } else { - // A new encoder waits for every old picture to be presented. + // A new encoder waits for every old picture to be presented. A closed + // track hands its last due time to whatever resumes after it. if closed.load(std::sync::atomic::Ordering::Relaxed) && buffers.lock().unwrap().len() == 1 { - break; + break queue.back().and_then(|frame| presentation.due(frame.timestamp)); } else if !queue.is_empty() { Step::Wait( queue @@ -320,7 +321,7 @@ impl Video { }; let Some((start, transition, before)) = next else { if closed.load(std::sync::atomic::Ordering::Relaxed) { - break; + break None; } self.changed.notified().await; continue; @@ -374,10 +375,11 @@ impl Video { .into_iter() .filter(|frame| floor.is_none_or(|floor| frame.timestamp.as_micros() >= floor.as_micros())), ); - } - Ok::<_, anyhow::Error>(()) + }; + Ok::<_, anyhow::Error>(drained) }; - tokio::try_join!(receive, decode)?; + let (_, drained) = tokio::try_join!(receive, decode)?; + role.retire(drained); Ok(()) } } @@ -740,6 +742,50 @@ mod tests { task.await.unwrap().unwrap(); } + #[tokio::test] + async fn a_closed_track_holds_its_end_until_its_last_picture_is_due() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let media = media(delay); + let broadcast = moq_net::broadcast::Info::new().produce(); + let source = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = source + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(delay)) + .await + .unwrap(); + let format = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let mut producer = moq_mux::container::Producer::new(source, format()); + for ms in [0, 33, 66] { + producer.write(frame(ms, ms == 0)).unwrap(); + } + // The publisher declares an end, then the track closes. + producer.discontinuity().unwrap(); + producer.finish().unwrap(); + let track = moq_mux::container::Consumer::new(subscriber, format()); + // The window never presents, so the decoded pictures stay queued. + let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0))); + while !task.is_finished() { + tokio::time::advance(Duration::from_millis(1)).await; + tokio::task::yield_now().await; + } + task.await.unwrap().unwrap(); + assert_eq!(media.video.lock().unwrap().len(), 3, "the pictures were presented"); + let last = media + .presentation + .lock() + .unwrap() + .due(moq_net::Timestamp::from_millis(66).unwrap()) + .unwrap(); + assert!(Instant::now().into_std() < last, "the track closed after its tail was due"); + let audio = Role::new(media.presentation.clone(), media.drained.clone(), super::super::playback::Kind::Audio); + let transition = audio.declare(); + audio.resume(transition, moq_net::Timestamp::ZERO, None).await.unwrap(); + assert!(Instant::now().into_std() >= last, "audio resumed over the video tail"); + } + #[tokio::test] async fn a_quick_resume_flushes_and_plays_every_old_picture_first() { tokio::time::pause(); From afa27539a80440122b95515a4c9813c9bbbb1465 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 15:44:08 -0700 Subject: [PATCH 18/22] test(play): scope the re-pin lock and format Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/timeline.rs | 9 +++++---- rs/moq-cli/src/play/video.rs | 11 +++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index 565b35c5ffe..25217403a33 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -838,10 +838,11 @@ mod tests { audio.end(ms(10_200)).unwrap(); audio.retire(Some((now + Duration::from_millis(300)).into_std())); // A replacement rendition starts over at zero and re-pins the anchor. - let mut pinned = presentation.lock().unwrap(); - pinned.restarted(); - pinned.audio(Duration::from_millis(50), DELAY, now.into_std()); - drop(pinned); + { + let mut presentation = presentation.lock().unwrap(); + presentation.restarted(); + presentation.audio(Duration::from_millis(50), DELAY, now.into_std()); + } let transition = video.declare(); video.plan(transition, ms(0), None); let resume = tokio::time::timeout(Duration::from_millis(400), video.resume(transition, ms(0), None)).await; diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 351cfc58872..66f58726530 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -779,8 +779,15 @@ mod tests { .unwrap() .due(moq_net::Timestamp::from_millis(66).unwrap()) .unwrap(); - assert!(Instant::now().into_std() < last, "the track closed after its tail was due"); - let audio = Role::new(media.presentation.clone(), media.drained.clone(), super::super::playback::Kind::Audio); + assert!( + Instant::now().into_std() < last, + "the track closed after its tail was due" + ); + let audio = Role::new( + media.presentation.clone(), + media.drained.clone(), + super::super::playback::Kind::Audio, + ); let transition = audio.declare(); audio.resume(transition, moq_net::Timestamp::ZERO, None).await.unwrap(); assert!(Instant::now().into_std() >= last, "audio resumed over the video tail"); From ef38e12330c5b1c9b8b66df368e3584bfa920444 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 16:11:22 -0700 Subject: [PATCH 19/22] fix(play): keep retired tails apart and arm the resume wakeup early A successor role's drop cleared the per-kind end slot, releasing its predecessor's retired tail. Retired tails now live in their own list. Resume also registers its change notification before reading the boundaries, so a change in between still wakes it. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/timeline.rs | 83 +++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index 25217403a33..698f4480fba 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -30,8 +30,13 @@ pub(super) struct Presentation { /// audio frame pins the anchor outright instead of pacing against media it /// will never play. restart: bool, - /// Endpoints declared by each role, on the shared playback timeline. + /// Endpoints declared by each live role, on the shared playback timeline. ends: [Option; 2], + /// When each retired role's tail finishes playing, frozen on the wall clock: + /// a later rendition re-pins the anchor, which would carry a media endpoint + /// onto an unrelated timeline. Kept apart from `ends` so a successor role + /// never clears its predecessor's tail. + tails: Vec, /// Encoder restarts whose replacements have not all started playing. transitions: std::collections::VecDeque, /// The latest transition each role declared. @@ -55,11 +60,9 @@ struct Pending { mapping: Option, } -enum Boundary { - /// A playing role's end on the shared timeline, which moves with the anchor. - Mapped { starts: Option, end: Duration }, - /// When a retired role's tail finishes playing, whatever the anchor does after. - Drained(Instant), +struct Boundary { + starts: Option, + end: Duration, } #[derive(Clone, Copy)] @@ -76,6 +79,7 @@ impl Presentation { speaker: false, restart: false, ends: Default::default(), + tails: Vec::new(), transitions: Default::default(), declared: Default::default(), next: Transition(1), @@ -201,8 +205,6 @@ pub(super) struct Role { changed: std::sync::Arc, role: Kind, mapping: std::sync::Mutex, - /// Whether the declared end was frozen for a tail that outlives the role. - retired: bool, } #[derive(Clone, Copy, Default)] @@ -243,21 +245,20 @@ impl Role { changed, role, mapping: Default::default(), - retired: false, } } /// Stop while a tail still plays until `drained`, holding a declared end until /// then. Dropping instead discards the tail with the end. - /// - /// The end is frozen on the wall clock: a later rendition re-pins the anchor, - /// which would carry a media timestamp onto an unrelated timeline. - pub(super) fn retire(mut self, drained: Option) { + pub(super) fn retire(self, drained: Option) { + let now = tokio::time::Instant::now().into_std(); let mut presentation = self.presentation.lock().unwrap(); - let end = &mut presentation.ends[self.role as usize]; - *end = end.take().and(drained.map(Boundary::Drained)); - drop(presentation); - self.retired = true; + presentation.tails.retain(|tail| *tail > now); + if presentation.ends[self.role as usize].take().is_some() + && let Some(drained) = drained + { + presentation.tails.push(drained); + } } pub(super) fn map(&self, source: Timestamp) -> anyhow::Result { @@ -312,7 +313,7 @@ impl Role { /// Mark where this role's current segment ends on the shared clock. pub(super) fn end(&self, source: Timestamp) -> anyhow::Result<()> { let mapping = *self.mapping.lock().unwrap(); - self.presentation.lock().unwrap().ends[self.role as usize] = Some(Boundary::Mapped { + self.presentation.lock().unwrap().ends[self.role as usize] = Some(Boundary { starts: mapping.starts, end: timestamp(mapping.map(source)?), }); @@ -373,6 +374,11 @@ impl Role { delay: Option, ) -> anyhow::Result { loop { + // Armed before reading the boundaries, so a change made between the read + // and the wait still wakes it. + let changed = self.changed.notified(); + let mut changed = std::pin::pin!(changed); + changed.as_mut().enable(); let now = tokio::time::Instant::now().into_std(); let end = { let mut presentation = self.presentation.lock().unwrap(); @@ -385,19 +391,15 @@ impl Role { .ends .iter() .flatten() - .filter_map(|boundary| match boundary { - Boundary::Mapped { starts, end } => starts - .is_none_or(|start| start <= now) - .then(|| presentation.due(media(*end))) - .flatten(), - Boundary::Drained(at) => Some(*at), - }) + .filter(|boundary| boundary.starts.is_none_or(|start| start <= now)) + .filter_map(|boundary| presentation.due(media(boundary.end))) + .chain(presentation.tails.iter().copied()) .max() }; if let Some(end) = end.filter(|end| *end > now) { tokio::select! { _ = tokio::time::sleep_until(end.into()) => {}, - _ = self.changed.notified() => {}, + _ = changed => {}, } continue; } @@ -440,10 +442,7 @@ impl Drop for Role { fn drop(&mut self) { let own = self.role as usize; if let Ok(mut presentation) = self.presentation.lock() { - // A retired end stays until it drains; nothing waits on it after that. - if !self.retired { - presentation.ends[own] = None; - } + presentation.ends[own] = None; for pending in presentation .transitions .iter_mut() @@ -849,6 +848,30 @@ mod tests { assert!(resume.is_ok(), "video waited past the drained tail"); } + #[tokio::test] + async fn a_cancelled_successor_keeps_a_retired_tail() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); + let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + audio.declare(); + audio.end(ms(10_200)).unwrap(); + audio.retire(Some((now + Duration::from_millis(300)).into_std())); + // The next rendition fails while the retired sink still plays. + let successor = Role::new(presentation.clone(), Default::default(), Kind::Audio); + successor.declare(); + drop(successor); + let transition = video.declare(); + video.plan(transition, ms(0), None); + let resume = tokio::time::timeout(Duration::from_millis(250), video.resume(transition, ms(0), None)).await; + assert!(resume.is_err(), "video resumed over the retired audio tail"); + } + #[tokio::test] async fn paired_plans_keep_media_offsets_and_follow_the_speakers_delay() { tokio::time::pause(); From e931d5c5b0df07d88608a6291d17f6c1b04d8635 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 17:23:47 -0700 Subject: [PATCH 20/22] fix(play): stop reading behind a held resume past the age budget While playback waits behind a resume, the audio PCM queue and the video segment queue grew with every arrival. Each receive loop now stops reading once media queued behind a resume arrived more than max_delay ago, leaving the excess to the subscription's own max_delay eviction. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/media.rs | 79 +++++++++++++++++++++++++++++++- rs/moq-cli/src/play/video.rs | 89 +++++++++++++++++++++++++++++++++++- 2 files changed, 165 insertions(+), 3 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index aadcad16993..0a1a4fef0fe 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -435,6 +435,7 @@ async fn play_audio( let role = Role::new(presentation.clone(), changed.clone(), Kind::Audio); let received = Mutex::new(VecDeque::new()); let arrived = tokio::sync::Notify::new(); + let popped = tokio::sync::Notify::new(); let max_delay = consumer.max_delay(); let sample_rate = consumer.sample_rate(); let layout = consumer.layout(); @@ -485,6 +486,17 @@ async fn play_audio( let mut current = None; let mut ends = VecDeque::new(); loop { + // Stop reading while play is held behind a resume with more than the age + // budget queued, so the subscription evicts the excess instead of this + // queue growing without bound. + while received + .lock() + .unwrap() + .front() + .is_some_and(|(.., queued): &(_, _, _, Instant)| queued.elapsed() > max_delay) + { + popped.notified().await; + } let event = match consumer.read().await { Err(err) if !matches!(err, moq_audio::Error::Decode(_)) => return Err(err.into()), event => event, @@ -524,7 +536,10 @@ async fn play_audio( } _ => None, }; - received.lock().unwrap().push_back((event, delay, crossing)); + received + .lock() + .unwrap() + .push_back((event, delay, crossing, Instant::now())); arrived.notify_one(); if ended { return Ok::<_, anyhow::Error>(()); @@ -538,8 +553,9 @@ async fn play_audio( let mut deferred = None; let mut sequential = false; loop { - let (event, delay, crossing) = loop { + let (event, delay, crossing, _) = loop { if let Some(event) = received.lock().unwrap().pop_front() { + popped.notify_one(); break event; } arrived.notified().await; @@ -1083,6 +1099,65 @@ mod tests { ); } + #[tokio::test] + async fn a_held_audio_resume_leaves_the_excess_to_the_subscription() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Audio)); + let mut options = moq_audio::decode::Options::new(); + options.delay = Some(delay); + options.max_delay = delay; + options.output.format = moq_audio::Format::F32; + let consumer = moq_audio::decode::Consumer::new( + &broadcast.consume(), + &AudioConfig::new(AudioCodec::Pcm, SAMPLE_RATE, 1), + "audio", + options, + ) + .await + .unwrap(); + let recorder = Recorder::default(); + let presentation = Arc::new(Mutex::new(Presentation::new(delay))); + let changed = Arc::new(tokio::sync::Notify::new()); + let playback = AudioPlayback { + changed: changed.clone(), + speaker: recorder.clone(), + presentation: presentation.clone(), + latency: delay, + output: recorder.clone(), + }; + let task = tokio::spawn(async move { + play_audio(consumer, playback).await.unwrap().finish().await; + }); + for index in 0..5 { + producer.write(packet(index, 0.25)).unwrap(); + tokio::time::sleep(PACKET_DURATION).await; + } + // Video declares an end two seconds out, holding the audio replacement back. + let video = Role::new(presentation.clone(), changed.clone(), Kind::Video); + video.declare(); + video.end(moq_net::Timestamp::from_millis(2_000).unwrap()).unwrap(); + producer.discontinuity().unwrap(); + const REPLACEMENT: u64 = 100; + for index in 0..REPLACEMENT { + producer.write(packet(5 + index, 0.5)).unwrap(); + tokio::time::sleep(PACKET_DURATION).await; + } + producer.finish().unwrap(); + task.await.unwrap(); + drop(video); + let (resumed, _) = played(&recorder, 0.5); + assert!(!resumed.is_zero(), "the replacement played"); + assert!( + resumed < PACKET_DURATION * REPLACEMENT as u32 / 2, + "media held behind the resume past the age budget was queued, not evicted: {resumed:?}" + ); + } + #[tokio::test] async fn an_evicted_replacement_gop_gives_way_to_the_next_replacement() { struct Passthrough; diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 66f58726530..2c4d6004279 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -76,6 +76,8 @@ impl Decoder for moq_video::decode::Sink { struct Segment { media: Buffer, resumes: Option, + /// When its first picture arrived. + arrived: Option, } impl std::ops::Deref for Segment { @@ -98,6 +100,8 @@ impl Video { let advancing = std::sync::atomic::AtomicBool::new(false); let revision = std::sync::atomic::AtomicU64::new(0); let buffers = Mutex::new(VecDeque::from([Segment::default()])); + // Signalled when a segment held behind a resume becomes the front. + let advanced = tokio::sync::Notify::new(); let role = Role::new( self.presentation.clone(), self.changed.clone(), @@ -110,6 +114,21 @@ impl Video { let mut end = None; loop { + // Stop reading while media queued behind a resume arrived more than the age + // budget ago, so the subscription evicts the excess instead of the queue + // growing a segment per restart without bound. + while buffers + .lock() + .unwrap() + .iter() + .skip(1) + .filter(|segment| segment.oldest().is_some()) + .filter_map(|segment| segment.arrived) + .min() + .is_some_and(|arrived| arrived.elapsed() > self.max_delay) + { + advanced.notified().await; + } let read = track.read().await?; let mut buffers = buffers.lock().unwrap(); let active = buffers.len() == 1 && !advancing.load(std::sync::atomic::Ordering::Relaxed); @@ -132,7 +151,9 @@ impl Video { { self.output.send(Event::Wake); } - buffers.back_mut().unwrap().push(frame, self.max_delay); + let back = buffers.back_mut().unwrap(); + back.push(frame, self.max_delay); + back.arrived.get_or_insert_with(Instant::now); false } // Applied in delivery order, so the restart lands before the frame after it. @@ -314,6 +335,7 @@ impl Video { Some((start, transition)) => { advancing.store(true, std::sync::atomic::Ordering::Relaxed); buffers.pop_front(); + advanced.notify_one(); Some((start, transition, revision.load(std::sync::atomic::Ordering::Relaxed))) } None => None, @@ -793,6 +815,71 @@ mod tests { assert!(Instant::now().into_std() >= last, "audio resumed over the video tail"); } + #[tokio::test] + async fn a_held_video_resume_leaves_the_excess_to_the_subscription() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let media = media(delay); + let broadcast = moq_net::broadcast::Info::new().produce(); + let source = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = source + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(delay)) + .await + .unwrap(); + let format = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let mut producer = moq_mux::container::Producer::new(source, format()); + let track = moq_mux::container::Consumer::new(subscriber, format()); + let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0))); + let audio = Role::new( + media.presentation.clone(), + media.drained.clone(), + super::super::playback::Kind::Audio, + ); + // Two-picture segments, one per restart, published live for two seconds + // while audio's end holds the first replacement back. + const SEGMENTS: u64 = 50; + let mut shown = 0; + // Presents until the track drains, which replaying every queued segment would + // take far longer than this bound. + for at in 0..60_000u64 { + if task.is_finished() { + break; + } + if at < 60 && at % 20 == 0 { + producer.write(frame(at, at == 0)).unwrap(); + } else if at == 60 { + audio.declare(); + audio.end(moq_net::Timestamp::from_millis(2_000).unwrap()).unwrap(); + } + if (60..60 + SEGMENTS * 40).contains(&at) && at % 20 == 0 { + if at % 40 == 20 { + producer.discontinuity().unwrap(); + } + producer.write(frame(at, at % 40 == 20)).unwrap(); + } else if at == 60 + SEGMENTS * 40 { + producer.finish().unwrap(); + } + tokio::task::yield_now().await; + if media + .output + .present(&media) + .is_some_and(|picture| picture.as_millis() >= 60) + { + shown += 1; + } + tokio::time::advance(Duration::from_millis(1)).await; + } + task.await.unwrap().unwrap(); + assert!(shown > 0, "the replacements played"); + assert!( + shown < SEGMENTS, + "pictures held behind the resume past the age budget were queued, not evicted: {shown}" + ); + } + #[tokio::test] async fn a_quick_resume_flushes_and_plays_every_old_picture_first() { tokio::time::pause(); From 3b066b9f7704b7b52021d1c695bb7009cfb486bd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 18:10:21 -0700 Subject: [PATCH 21/22] fix(play): discard emptied superseded video segments A replacement segment that eviction emptied never receives another picture once a newer one starts, yet it was kept with its transition and skipped by the backlog bound. Receive now drops it and abandons its transition when the next replacement arrives. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/timeline.rs | 6 ++++ rs/moq-cli/src/play/video.rs | 64 +++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index 698f4480fba..6b15184b8f4 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -87,6 +87,12 @@ impl Presentation { } } + /// How many transitions are still pending. + #[cfg(test)] + pub(super) fn pending(&self) -> usize { + self.transitions.len() + } + fn transition(&mut self, id: Transition) -> Option<&mut Pending> { self.transitions.iter_mut().find(|pending| pending.id == id) } diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 2c4d6004279..5b0455e3a45 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -136,6 +136,17 @@ impl Video { let ended = match read { Some(Read::Frame { group, frame }) => { if let Some((_, transition)) = end.take_if(|(end, _)| *end != group) { + // A superseded segment never receives another picture, so one that + // eviction emptied can never resume. + let mut index = 1; + while index < buffers.len() { + if buffers[index].oldest().is_some() { + index += 1; + } else if let Some(skipped) = buffers.remove(index).and_then(|skipped| skipped.resumes) + { + role.abandon(skipped); + } + } role.plan(transition, frame.timestamp, None); buffers.push_back(Segment { resumes: Some(transition), @@ -880,6 +891,59 @@ mod tests { ); } + #[tokio::test] + async fn an_emptied_superseded_segment_is_discarded() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let media = media(delay); + let broadcast = moq_net::broadcast::Info::new().produce(); + let source = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + // The network keeps every group; only the player's age budget evicts. + let subscriber = source + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))) + .await + .unwrap(); + let format = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let mut producer = moq_mux::container::Producer::new(source, format()); + let track = moq_mux::container::Consumer::new(subscriber, format()); + let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0))); + // Audio's end two seconds out holds every replacement back. + let audio = Role::new( + media.presentation.clone(), + media.drained.clone(), + super::super::playback::Kind::Audio, + ); + audio.declare(); + audio.end(moq_net::Timestamp::from_millis(2_000).unwrap()).unwrap(); + producer.write(frame(0, true)).unwrap(); + // Each replacement is one GOP longer than the age budget, so eviction + // takes its only keyframe and leaves it empty. + const SEGMENTS: u64 = 10; + for segment in 0..SEGMENTS { + producer.discontinuity().unwrap(); + for index in 0..15 { + producer + .write(frame(20 + segment * 300 + index * 20, index == 0)) + .unwrap(); + } + } + tokio::time::advance(Duration::from_millis(500)).await; + tokio::task::yield_now().await; + let pending = media.presentation.lock().unwrap().pending(); + assert!(pending <= 3, "emptied segments kept their transitions: {pending}"); + producer.finish().unwrap(); + drop(audio); + for _ in 0..1_000 { + tokio::task::yield_now().await; + media.output.present(&media); + tokio::time::advance(Duration::from_millis(1)).await; + } + task.await.unwrap().unwrap(); + } + #[tokio::test] async fn a_quick_resume_flushes_and_plays_every_old_picture_first() { tokio::time::pause(); From 311cb4a080d77f9e6aeeb7be562dcc145f6863f6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 16 Oct 2026 18:25:31 -0700 Subject: [PATCH 22/22] fix(play): give resumes their own timeline wakeup A resume waiting on a boundary shared one Notify with the video decoder, so a notify_one meant for the decoder could wake the resume instead and leave a new picture undecoded until the boundary passed. Presentation now owns a timeline Notify that only resumes wait on, woken on boundary, transition and anchor changes. Role no longer takes the shared notifier. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/media.rs | 4 +- rs/moq-cli/src/play/timeline.rs | 95 ++++++++++++++++++--------------- rs/moq-cli/src/play/video.rs | 93 +++++++++++++++++++++++++------- 3 files changed, 128 insertions(+), 64 deletions(-) diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 0a1a4fef0fe..d0a19b62c85 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -432,7 +432,7 @@ async fn play_audio( // until the old sink drains. The window // schedules video against where the speaker actually is, which keeps the two // together whatever the target does. - let role = Role::new(presentation.clone(), changed.clone(), Kind::Audio); + let role = Role::new(presentation.clone(), Kind::Audio); let received = Mutex::new(VecDeque::new()); let arrived = tokio::sync::Notify::new(); let popped = tokio::sync::Notify::new(); @@ -1138,7 +1138,7 @@ mod tests { tokio::time::sleep(PACKET_DURATION).await; } // Video declares an end two seconds out, holding the audio replacement back. - let video = Role::new(presentation.clone(), changed.clone(), Kind::Video); + let video = Role::new(presentation.clone(), Kind::Video); video.declare(); video.end(moq_net::Timestamp::from_millis(2_000).unwrap()).unwrap(); producer.discontinuity().unwrap(); diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index 6b15184b8f4..d4e8d330e55 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -43,6 +43,10 @@ pub(super) struct Presentation { declared: [Transition; 2], next: Transition, reference: Option, + /// Wakes every pending resume when a boundary, a transition or the anchor + /// changes. Kept apart from the window's and the decoders' wakeups, so a + /// resume can never take a wakeup meant for them. + timeline: std::sync::Arc, } /// One encoder restart, shared by the roles whose ends declared it. @@ -84,6 +88,7 @@ impl Presentation { declared: Default::default(), next: Transition(1), reference: None, + timeline: Default::default(), } } @@ -132,13 +137,18 @@ impl Presentation { self.reference = Some(timestamp); let before = self.pacer.due(timestamp); - before != Some(self.pacer.pace(timestamp, now)) + let moved = before != Some(self.pacer.pace(timestamp, now)); + if moved { + self.timeline.notify_waiters(); + } + moved } /// Start a new video timeline without taking ownership from a speaker. pub(super) fn video_restarted(&mut self) { if !self.speaker { self.pacer = moq_mux::Pacer::default().with_delay(self.delay); + self.timeline.notify_waiters(); } } @@ -175,7 +185,11 @@ impl Presentation { }; self.speaker = true; - before != Some(after) + let moved = before != Some(after); + if moved { + self.timeline.notify_waiters(); + } + moved } /// The speaker restarted on a new timeline, so the media either side of the @@ -208,7 +222,7 @@ impl Presentation { /// Maps each encoder segment onto the shared playback clock. pub(super) struct Role { presentation: std::sync::Arc>, - changed: std::sync::Arc, + timeline: std::sync::Arc, role: Kind, mapping: std::sync::Mutex, } @@ -241,14 +255,11 @@ impl Mapping { } impl Role { - pub(super) fn new( - presentation: std::sync::Arc>, - changed: std::sync::Arc, - role: Kind, - ) -> Self { + pub(super) fn new(presentation: std::sync::Arc>, role: Kind) -> Self { + let timeline = presentation.lock().unwrap().timeline.clone(); Self { presentation, - changed, + timeline, role, mapping: Default::default(), } @@ -312,7 +323,7 @@ impl Role { }); presentation.declared[own] = id; drop(presentation); - self.changed.notify_waiters(); + self.timeline.notify_waiters(); id } @@ -323,7 +334,7 @@ impl Role { starts: mapping.starts, end: timestamp(mapping.map(source)?), }); - self.changed.notify_waiters(); + self.timeline.notify_waiters(); Ok(()) } @@ -338,13 +349,13 @@ impl Role { }); } drop(presentation); - self.changed.notify_waiters(); + self.timeline.notify_waiters(); } /// Media was lost: nothing waits for the discarded tail. pub(super) fn gap(&self) { self.presentation.lock().unwrap().ends[self.role as usize] = None; - self.changed.notify_waiters(); + self.timeline.notify_waiters(); } /// This role will never resume `transition`, so a peer must not pair with its plan. @@ -357,7 +368,7 @@ impl Role { } presentation.prune(tokio::time::Instant::now().into_std()); drop(presentation); - self.changed.notify_waiters(); + self.timeline.notify_waiters(); } pub(super) fn apply(&self, transition: Transition, mapping: Mapping) { @@ -368,7 +379,7 @@ impl Role { } presentation.prune(tokio::time::Instant::now().into_std()); drop(presentation); - self.changed.notify_waiters(); + self.timeline.notify_waiters(); } /// Drain declared old media, then map `transition`'s replacement onto the @@ -382,7 +393,7 @@ impl Role { loop { // Armed before reading the boundaries, so a change made between the read // and the wait still wakes it. - let changed = self.changed.notified(); + let changed = self.timeline.notified(); let mut changed = std::pin::pin!(changed); changed.as_mut().enable(); let now = tokio::time::Instant::now().into_std(); @@ -460,7 +471,7 @@ impl Drop for Role { presentation.declared[own] = Transition::default(); presentation.prune(tokio::time::Instant::now().into_std()); } - self.changed.notify_waiters(); + self.timeline.notify_waiters(); } } @@ -584,8 +595,8 @@ mod tests { tokio::time::pause(); let start = tokio::time::Instant::now(); let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); - let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); - let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); presentation.lock().unwrap().video(ms(10_000), start.into_std()); let transition = audio.declare(); audio.end(ms(10_020)).unwrap(); @@ -606,8 +617,8 @@ mod tests { .lock() .unwrap() .audio(Duration::from_secs(10), DELAY, now.into_std()); - let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); - let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); let transition = audio.declare(); assert_eq!(video.declare(), transition, "one restart"); audio.end(ms(10_020)).unwrap(); @@ -634,8 +645,8 @@ mod tests { .lock() .unwrap() .audio(Duration::from_secs(10), DELAY, now.into_std()); - let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); - let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); let transition = audio.declare(); video.declare(); audio.end(ms(10_020)).unwrap(); @@ -665,8 +676,8 @@ mod tests { .lock() .unwrap() .audio(Duration::from_secs(10), DELAY, now.into_std()); - let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); - let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); let transition = audio.declare(); video.declare(); audio.end(ms(10_020)).unwrap(); @@ -697,8 +708,8 @@ mod tests { .lock() .unwrap() .audio(Duration::from_secs(10), DELAY, now.into_std()); - let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); - let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); let transition = audio.declare(); video.declare(); audio.end(ms(10_020)).unwrap(); @@ -729,8 +740,8 @@ mod tests { .lock() .unwrap() .audio(Duration::from_secs(10), DELAY, now.into_std()); - let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); - let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); let transition = audio.declare(); video.declare(); audio.end(ms(10_020)).unwrap(); @@ -756,8 +767,8 @@ mod tests { .lock() .unwrap() .audio(Duration::from_secs(10), DELAY, now.into_std()); - let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); - let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); let transition = audio.declare(); video.declare(); audio.end(ms(10_020)).unwrap(); @@ -785,8 +796,8 @@ mod tests { .lock() .unwrap() .audio(Duration::from_secs(10), DELAY, now.into_std()); - let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); - let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); audio.declare(); audio.end(ms(10_200)).unwrap(); // Cancelled with its sink, so nothing plays the tail. @@ -811,8 +822,8 @@ mod tests { .lock() .unwrap() .audio(Duration::from_secs(10), DELAY, now.into_std()); - let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); - let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); audio.declare(); audio.end(ms(10_200)).unwrap(); // The track closed, but its returned sink still plays the tail up to the end. @@ -837,8 +848,8 @@ mod tests { .lock() .unwrap() .audio(Duration::from_secs(10), DELAY, now.into_std()); - let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); - let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); audio.declare(); audio.end(ms(10_200)).unwrap(); audio.retire(Some((now + Duration::from_millis(300)).into_std())); @@ -863,13 +874,13 @@ mod tests { .lock() .unwrap() .audio(Duration::from_secs(10), DELAY, now.into_std()); - let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); - let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); audio.declare(); audio.end(ms(10_200)).unwrap(); audio.retire(Some((now + Duration::from_millis(300)).into_std())); // The next rendition fails while the retired sink still plays. - let successor = Role::new(presentation.clone(), Default::default(), Kind::Audio); + let successor = Role::new(presentation.clone(), Kind::Audio); successor.declare(); drop(successor); let transition = video.declare(); @@ -887,8 +898,8 @@ mod tests { .lock() .unwrap() .audio(Duration::from_secs(10), DELAY, now.into_std()); - let audio = Role::new(presentation.clone(), Default::default(), Kind::Audio); - let video = Role::new(presentation.clone(), Default::default(), Kind::Video); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); let transition = audio.declare(); video.declare(); audio.end(ms(10_020)).unwrap(); diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 5b0455e3a45..9d7d1794026 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -102,11 +102,7 @@ impl Video { let buffers = Mutex::new(VecDeque::from([Segment::default()])); // Signalled when a segment held behind a resume becomes the front. let advanced = tokio::sync::Notify::new(); - let role = Role::new( - self.presentation.clone(), - self.changed.clone(), - super::playback::Kind::Video, - ); + let role = Role::new(self.presentation.clone(), super::playback::Kind::Video); let receive = async { // The group of a declared end whose restart waits for a later group, so the // media buffered before the end still plays out, and the transition that @@ -816,11 +812,7 @@ mod tests { Instant::now().into_std() < last, "the track closed after its tail was due" ); - let audio = Role::new( - media.presentation.clone(), - media.drained.clone(), - super::super::playback::Kind::Audio, - ); + let audio = Role::new(media.presentation.clone(), super::super::playback::Kind::Audio); let transition = audio.declare(); audio.resume(transition, moq_net::Timestamp::ZERO, None).await.unwrap(); assert!(Instant::now().into_std() >= last, "audio resumed over the video tail"); @@ -844,11 +836,7 @@ mod tests { let mut producer = moq_mux::container::Producer::new(source, format()); let track = moq_mux::container::Consumer::new(subscriber, format()); let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0))); - let audio = Role::new( - media.presentation.clone(), - media.drained.clone(), - super::super::playback::Kind::Audio, - ); + let audio = Role::new(media.presentation.clone(), super::super::playback::Kind::Audio); // Two-picture segments, one per restart, published live for two seconds // while audio's end holds the first replacement back. const SEGMENTS: u64 = 50; @@ -911,11 +899,7 @@ mod tests { let track = moq_mux::container::Consumer::new(subscriber, format()); let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0))); // Audio's end two seconds out holds every replacement back. - let audio = Role::new( - media.presentation.clone(), - media.drained.clone(), - super::super::playback::Kind::Audio, - ); + let audio = Role::new(media.presentation.clone(), super::super::playback::Kind::Audio); audio.declare(); audio.end(moq_net::Timestamp::from_millis(2_000).unwrap()).unwrap(); producer.write(frame(0, true)).unwrap(); @@ -944,6 +928,75 @@ mod tests { task.await.unwrap().unwrap(); } + #[tokio::test] + async fn a_waiting_resume_leaves_the_decoders_wakeup_alone() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let media = media(delay); + let broadcast = moq_net::broadcast::Info::new().produce(); + let source = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = source + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(delay)) + .await + .unwrap(); + let format = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let mut producer = moq_mux::container::Producer::new(source, format()); + let track = moq_mux::container::Consumer::new(subscriber, format()); + let decoder = Buffered::new(0); + let decoded = decoder.decoded.clone(); + let task = tokio::spawn(playback(&media, delay).run(track, decoder)); + for ms in [0, 20, 40] { + producer.write(frame(ms, ms == 0)).unwrap(); + } + // Present until `count` pictures are decoded, or give up after 20 s. + let step = async |count: usize| { + for _ in 0..20_000 { + if decoded.lock().unwrap().len() >= count { + break; + } + tokio::task::yield_now().await; + media.output.present(&media); + tokio::time::advance(Duration::from_millis(1)).await; + } + }; + step(3).await; + // Audio waits on an end ten seconds out while the decoder idles. + let audio = Role::new(media.presentation.clone(), super::super::playback::Kind::Audio); + let transition = audio.declare(); + audio.end(moq_net::Timestamp::from_secs(10).unwrap()).unwrap(); + let resume = tokio::spawn(async move { + audio + .resume(transition, moq_net::Timestamp::ZERO, None) + .await + .map(|_| ()) + }); + for _ in 0..200 { + tokio::task::yield_now().await; + media.output.present(&media); + tokio::time::advance(Duration::from_millis(1)).await; + } + // The publisher sends one more picture after the pause. + producer.write(frame(60, false)).unwrap(); + let sent = Instant::now(); + step(4).await; + assert!( + sent.elapsed() < Duration::from_millis(50), + "the decoder missed the new picture for {:?}", + sent.elapsed() + ); + producer.finish().unwrap(); + resume.abort(); + for _ in 0..500 { + tokio::task::yield_now().await; + media.output.present(&media); + tokio::time::advance(Duration::from_millis(1)).await; + } + task.await.unwrap().unwrap(); + } + #[tokio::test] async fn a_quick_resume_flushes_and_plays_every_old_picture_first() { tokio::time::pause();