diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index 48cf0193667..42ba4b876fe 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -35,6 +35,12 @@ These land with the next breaking release, not the 2026-09-23 train. `catalog::hang::Consumer` with `.into()`, then read it through `catalog::Stream`. Its `Hang`, `Msf`, and `Closed` variants and `catalog::msf::Consumer::track` are private. +- **Rust audio reads carry encoder ends.** `moq_audio::decode::Consumer::read()` + returns `decode::Read` events: `Frame { discontinuity, frame }` and + `End { discontinuity, end }`; `None` still means the track is finished. An + `End` can be followed by final PCM with the same `discontinuity` before a fresh + encoder bumps it. A bump with no `End` before it is a loss. + Binding audio frame APIs keep their existing shape. - **JS catalog ranking names its media kind.** Replace `Catalog.ranked(renditions)` with `Catalog.Video.ranked(renditions)`. `Catalog.Audio.ranked(renditions)` orders audio by enabled state, bitrate, sample rate, channel count, and name. diff --git a/quest/m2/README.md b/quest/m2/README.md index fbd839f065b..4bed2282a49 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -183,6 +183,5 @@ ranked just below deterministic TS legs, which it requires. - [Publisher timeliness at relay ingest](/quest/m2/qos-publisher-timeliness.md) - follow-on capability after the release contract is stable -- [Play resume](/quest/m2/play-resume.md) - `moq-cli play` plays out pictures buffered before a declared end when new media arrives quickly - [Task supervision](/quest/m2/task-supervision.md) - the auth decider loop, stats publisher, and moq-tokio's token and WebSocket internals are supervised like #5447's tasks - [Auth serve bound address](/quest/m2/auth-serve-bound-addr.md) - `moq auth serve` logs the port it actually bound diff --git a/quest/m2/play-resume.md b/quest/m2/play-resume.md deleted file mode 100644 index 7a2b63dc80c..00000000000 --- a/quest/m2/play-resume.md +++ /dev/null @@ -1,31 +0,0 @@ -# [M] moq-cli play plays out pictures before a quick resume - -## Goal - -When new media arrives before the pictures buffered ahead of a declared end -have played (a source switch, say), `moq-cli play` plays the old timeline out -first instead of dropping it. - -## Plan - -#5288 fixed `rs/moq-cli/src/play/video.rs` so `Read::End` no longer discards -the buffered pictures, with the restart waiting for the first frame of a later -group. A resume that arrives before those pictures play still drops them, as -it did before #5288. - -Playing them out needs a sequential timeline in `play`: queue the new group's -frames behind the remaining pre-end pictures, then rebase the clock at the -boundary. Queueing encoded frames is not enough when the decoder reorders: the -declared end has to reach the decode loop so it flushes the pictures it still -holds before decoding the resumed group, which today happens only when the -track ends. - -✅ Decided 2026-10-13: audio queues with video. One sequential timeline: audio -and video both play out the old segment, then rebase together, which keeps -A/V sync. Not chosen: audio snaps to the new timeline; leave it open. - -Test with a declared end followed immediately by a new group, using a decoder -that holds pictures (depth above zero): every pre-end picture is shown before -the first new one. - -Public API: none. Wire: none. diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index d2b30e0eb1e..ccafa680887 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -2,7 +2,7 @@ use std::collections::VecDeque; -use moq_mux::container::Read; +use moq_mux::container::Read as Encoded; use std::time::{Duration, Instant}; use bytes::Bytes; @@ -12,6 +12,30 @@ use crate::jitter::{self, Target}; use crate::resample::{Remix, Resampler}; use crate::{Activity, Error, Format, Frame, Layout}; +/// Decoded PCM and encoder ends in delivery order. +/// +/// `discontinuity` goes up wherever the decoded timeline breaks: at the fresh +/// encoder after an [`End`](Self::End), or at a loss. A bump with no `End` +/// before it is a loss, and the hole runs from the previous frame's end to the +/// new frame's timestamp. +#[derive(Clone, Debug)] +pub enum Read { + /// Decoded PCM, including the drained tail before a discontinuity. + Frame { + /// The decoded timeline these samples belong to; only changes matter. + discontinuity: u64, + /// The decoded samples and their presentation time. + frame: Frame, + }, + /// The publisher stopped this encoder; its final PCM may still follow. + End { + /// The decoded timeline holding the stopped encoder's final samples. + discontinuity: u64, + /// Samples at or past this point are padding and are trimmed. + end: moq_net::Timestamp, + }, +} + /// Where a consumer starts on a track that already holds groups. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[non_exhaustive] @@ -68,8 +92,7 @@ impl Options { /// /// The mirror of [`encode::Producer`](crate::encode::Producer): output format / /// sample rate / layout are fixed at construction, and -/// [`read`](Self::read) returns [`Frame`]s carrying the codec activity they -/// were decoded from. +/// [`read`](Self::read) returns [`Read`] events carrying decoded samples and track boundaries. pub struct Consumer { decoder: Decoder, track: moq_mux::container::Consumer, @@ -92,7 +115,9 @@ pub struct Consumer { next_start: Option, /// Frames decoded and not yet handed back, so a gap's tail can be returned /// ahead of the packet that exposed it. - ready: VecDeque, + ready: VecDeque, + /// Stable across all groups and the resampler tail of one decoded timeline. + discontinuity: u64, /// Codec activity spans the resampler's buffered output still covers. spans: VecDeque, /// Activity of the last span the output ran past, for the rounding samples the @@ -209,6 +234,7 @@ impl Consumer { resolved_layout: layout, next_start: None, ready: VecDeque::new(), + discontinuity: 0, spans: VecDeque::new(), trailing: Activity::Active, epoch: None, @@ -258,7 +284,7 @@ impl Consumer { self.resolved_layout } - /// Read the next decoded PCM frame, or `None` when the track ends. + /// Read decoded PCM or a track boundary, or `None` when the track ends. /// /// [`Frame::activity`] reports whether the packet these samples came from /// coded audio. It describes where the frame begins, so a resampled @@ -270,34 +296,43 @@ impl Consumer { /// side stay anchored to their own packet timeline, so the hole is there to /// see. "Doesn't continue" allows for the quantization the stamps carry, which /// on a millisecond-stamped ingest is most of a millisecond. - pub async fn read(&mut self) -> Result, Error> { + pub async fn read(&mut self) -> Result, Error> { loop { if let Some(frame) = self.ready.pop_front() { return Ok(Some(frame)); } let mux_frame = match self.track.read().await? { - Some(Read::Frame { group, frame }) => { + Some(Encoded::Frame { group, frame }) => { // A later group after a declared end comes from a fresh encoder; the end's // own group carries the stopped encoder's final packets, trimmed below. if self.end.is_some() && group != self.end_group { self.restart()?; + self.discontinuity += 1; } frame } - Some(Read::Gap { .. }) => { + // The loss shows as the next frame's discontinuity, with no end before it. + Some(Encoded::Gap { .. }) => { self.restart()?; + self.discontinuity += 1; self.warmup_pending = !self.warmup.is_zero(); continue; } - Some(Read::End { group, end }) => { + Some(Encoded::End { group, end }) => { self.end = Some(end); self.end_group = group; self.frames_decoded = 0; self.terminal_start = None; - continue; + return Ok(Some(Read::End { + discontinuity: self.discontinuity, + end, + })); + } + None => { + let discontinuity = self.discontinuity; + return Ok(self.flush()?.map(|frame| Read::Frame { discontinuity, frame })); } - None => return self.flush(), }; // Every packet the container hands over is an arrival, including one the @@ -323,9 +358,14 @@ impl Consumer { && self .next_start .is_some_and(|next| discontinuous(next, mux_frame.timestamp)) - && let Some(frame) = self.gap()? { - self.ready.push_back(frame); + if let Some(frame) = self.gap()? { + self.ready.push_back(Read::Frame { + discontinuity: self.discontinuity, + frame, + }); + } + self.discontinuity += 1; } let rate = self.decoder.sample_rate(); @@ -427,7 +467,10 @@ impl Consumer { // Queued rather than returned, so a tail drained at a gap earlier in this // same iteration still comes out first. The next turn of the loop pops it. if let Some(frame) = self.frame(pcm, timestamp, activity)? { - self.ready.push_back(frame); + self.ready.push_back(Read::Frame { + discontinuity: self.discontinuity, + frame, + }); } } } @@ -439,7 +482,10 @@ impl Consumer { /// as their own frame first rather than filtered together with what follows it. fn restart(&mut self) -> Result<(), Error> { if let Some(frame) = self.drain()? { - self.ready.push_back(frame); + self.ready.push_back(Read::Frame { + discontinuity: self.discontinuity, + frame, + }); } self.next_start = None; self.spans.clear(); @@ -648,6 +694,16 @@ fn rewind(timestamp: moq_net::Timestamp, frames: usize, sample_rate: u32) -> Res #[cfg(test)] mod tests { + async fn next_frame(consumer: &mut Consumer) -> Result, Error> { + loop { + match consumer.read().await? { + Some(Read::Frame { frame, .. }) => return Ok(Some(frame)), + Some(Read::End { .. }) => {} + None => return Ok(None), + } + } + } + use moq_net::Timestamp; use super::*; @@ -696,7 +752,7 @@ mod tests { .unwrap(); let mut stamps = Vec::new(); let mut frames = 0; - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = next_frame(&mut consumer).await.unwrap() { stamps.push(frame.timestamp.as_micros()); frames += frame.data.len() / size_of::(); } @@ -765,7 +821,7 @@ mod tests { ) .await .unwrap(); - let frame = consumer.read().await.unwrap().unwrap(); + let frame = next_frame(&mut consumer).await.unwrap().unwrap(); assert!(frame.timestamp.as_micros() >= 1_080_000); let first = f32::from_le_bytes(frame.data[..4].try_into().unwrap()); assert!( @@ -814,9 +870,9 @@ mod tests { ) .await .unwrap(); - assert!(consumer.read().await.is_err()); + assert!(next_frame(&mut consumer).await.is_err()); let mut stamps = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = next_frame(&mut consumer).await.unwrap() { stamps.push(frame.timestamp.as_micros()); } assert_eq!(stamps, [100_000, 120_000]); @@ -866,7 +922,7 @@ mod tests { } producer.finish().unwrap(); let mut stamps = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = next_frame(&mut consumer).await.unwrap() { stamps.push(frame.timestamp.as_micros()); } assert_eq!(stamps.len(), 4, "each span's body and tail: {stamps:?}"); @@ -920,7 +976,7 @@ mod tests { } producer.write(&Frame::new(data.into(), Timestamp::ZERO)).unwrap(); - let frame = consumer.read().await.unwrap().expect("decoded frame"); + let frame = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); let samples = Format::F32.as_interleaved_f32(&frame.data, 2).unwrap(); assert_eq!(samples.len(), (960 - 312) * 2); for pair in samples.as_chunks::<2>().0.iter() { @@ -964,12 +1020,114 @@ mod tests { // 312 samples at 48 kHz is 6.5 ms. for (micros, frames) in [(0, 960 - 312), (13_500, 960), (33_500, 960)] { - let frame = consumer.read().await.unwrap().expect("decoded frame"); + let frame = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); assert_eq!(frame.timestamp.as_micros(), micros); assert_eq!(frame.data.len() / size_of::(), frames); } } + #[tokio::test] + async fn an_undeclared_timestamp_hole_is_a_discontinuity() { + tokio::time::pause(); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 48_000, 1); + let options = Options { + max_delay: Duration::from_secs(1), + ..Options::new() + }; + let mut consumer = Consumer::new(&broadcast.consume(), &catalog, "audio", options) + .await + .unwrap(); + // Contiguous groups, so the container sees no gap: only the stamps jump. + for micros in [0, 20_000, 500_000] { + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_micros(micros).unwrap(), + duration: Some(Timestamp::from_micros(20_000).unwrap()), + keyframe: true, + payload: (0..960).flat_map(|_| 0.25f32.to_le_bytes()).collect::>().into(), + }) + .unwrap(); + } + producer.finish().unwrap(); + let mut timelines = Vec::new(); + while let Some(event) = consumer.read().await.unwrap() { + if let Read::Frame { discontinuity, frame } = event { + timelines.push((frame.timestamp.as_micros(), discontinuity)); + } + } + assert_eq!(timelines, [(0, 0), (20_000, 0), (500_000, 1)]); + } + + #[tokio::test] + async fn decoded_segments_keep_resampler_tails_before_fresh_encoder_frames() { + tokio::time::pause(); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 44_100, 1); + let mut consumer = Consumer::new( + &broadcast.consume(), + &catalog, + "audio", + Options { + output: Output { + sample_rate: Some(48_000), + ..Output::default() + }, + max_delay: Duration::from_secs(1), + ..Options::new() + }, + ) + .await + .unwrap(); + for (micros, sample) in [(0, 0.25f32), (30_000, 0.5)] { + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_micros(micros).unwrap(), + duration: Some(Timestamp::from_scale(1024, 44_100).unwrap()), + keyframe: true, + payload: (0..1024).flat_map(|_| sample.to_le_bytes()).collect::>().into(), + }) + .unwrap(); + if micros == 0 { + producer.discontinuity().unwrap(); + } + } + producer.finish().unwrap(); + let mut ended = None; + let mut tail = false; + let mut resumed = false; + while let Some(event) = consumer.read().await.unwrap() { + match event { + Read::End { discontinuity, .. } if ended.is_none() => ended = Some(discontinuity), + Read::Frame { discontinuity, .. } if ended == Some(discontinuity) => { + assert!(!resumed); + tail = true; + } + Read::Frame { discontinuity, .. } if ended.is_some_and(|old| old != discontinuity) => resumed = true, + _ => {} + } + } + assert!( + tail, + "the old resampler tail must follow its End without looking like a new encoder" + ); + assert!(resumed, "a later group opens a fresh decoded timeline"); + } + /// A packet whose sample count isn't a multiple of the resampler's chunk leaves /// samples buffered, and the next output starts with those. Stamping that /// output with the packet that completed the chunk puts it up to a chunk late, @@ -1020,7 +1178,7 @@ mod tests { .unwrap(); } - let first = consumer.read().await.unwrap().expect("decoded frame"); + let first = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); assert_eq!(first.timestamp.as_micros(), 0); // Continuity, not a fixed number: the second frame starts where the first @@ -1028,7 +1186,7 @@ mod tests { // exactly, because the resampler emits whole frames and its count per chunk // wobbles around the nominal ratio; a real hole (the samples it held back, or // the startup silence it dropped) is twenty times this tolerance. - let second = consumer.read().await.unwrap().expect("decoded frame"); + let second = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); let first_frames = (first.data.len() / size_of::()) as u128; let ends_at = first_frames * 1_000_000 / 48_000; let gap = second.timestamp.as_micros().abs_diff(ends_at); @@ -1080,10 +1238,10 @@ mod tests { .unwrap(); producer.finish().unwrap(); - let first = consumer.read().await.unwrap().expect("decoded frame"); + let first = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); let first_frames = first.data.len() / size_of::(); - let tail = consumer.read().await.unwrap().expect("flushed tail"); + let tail = next_frame(&mut consumer).await.unwrap().expect("flushed tail"); let tail_frames = tail.data.len() / size_of::(); // The 142 held-back frames at 44.1 kHz are ~155 at 48 kHz, plus the 69 the @@ -1101,7 +1259,7 @@ mod tests { // what the drain adds here is what the start dropped off the front. let total = first_frames + tail_frames; assert!((1105..=1120).contains(&total), "unexpected total: {total}"); - assert!(consumer.read().await.unwrap().is_none()); + assert!(next_frame(&mut consumer).await.unwrap().is_none()); } #[tokio::test] @@ -1163,7 +1321,7 @@ mod tests { let expected = first_dtx.expect("silence should enter Opus DTX"); let mut actual = None; - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = next_frame(&mut consumer).await.unwrap() { // 10 ms packets do not fill the 20 ms chunk, so the resampler hands back // nothing every other packet. Those must not surface as frames: a frame // with no samples reads as audio arriving, and carries an activity @@ -1237,7 +1395,7 @@ mod tests { producer.finish().unwrap(); let mut read = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { + while let Some(frame) = next_frame(&mut consumer).await.unwrap() { read.push((frame.timestamp.as_micros(), frame.data.len() / size_of::())); } read @@ -1387,7 +1545,7 @@ mod tests { write(&mut producer, 48_000, dtx.payload, false); producer.finish().unwrap(); - let frame = consumer.read().await.unwrap().expect("decoded frame"); + let frame = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); // The output begins with the first packet's samples, so it is stamped and // labelled from that packet. Rewinding from the terminal one instead drops it // most of a second into the future, carrying the DTX label with it. @@ -1492,17 +1650,20 @@ mod tests { // The pre-skip is trimmed off the first packet, so it decodes short. That // shortfall is codec delay, not a hole: without counting it the packet after // every stream start would read as a gap. - let first = consumer.read().await.unwrap().expect("decoded frame"); + let first = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); let frames = first.data.len() / size_of::(); assert!(frames < 960, "the pre-skip should be trimmed, got {frames} frames"); // The hole is real, so codec prediction starts over but stream-level pre-skip // does not. The audio is stamped where the packet says rather than 2.5 ms early. - let second = consumer.read().await.unwrap().expect("decoded frame"); + let second = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); assert_eq!(second.timestamp.as_micros(), 22_500); assert_eq!(second.data.len() / size_of::(), 960, "pre-skip was reapplied"); - let third = consumer.read().await.unwrap().expect("decoded frame after gap"); + let third = next_frame(&mut consumer) + .await + .unwrap() + .expect("decoded frame after gap"); let second_frames = second.data.len() / size_of::(); assert_eq!( third.timestamp, @@ -1569,7 +1730,10 @@ mod tests { .unwrap(); producer.cut(None).unwrap(); - consumer.read().await.unwrap().expect("one decoded frame per packet"); + next_frame(&mut consumer) + .await + .unwrap() + .expect("one decoded frame per packet"); series.push(crate::jitter::millis(consumer.delay())); } CLOCK.set(None); @@ -1640,7 +1804,7 @@ mod tests { .unwrap(); producer.cut(None).unwrap(); - let frame = consumer.read().await.unwrap().expect("decoded frame"); + let frame = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); assert!( frame.data.len() / size_of::() < encoder.frame_size(), "pre-skip is trimmed" @@ -1747,8 +1911,8 @@ mod tests { producer.cut(None).unwrap(); } - let first = consumer.read().await.unwrap().expect("first decoded frame"); - let second = consumer.read().await.unwrap().expect("second decoded frame"); + let first = next_frame(&mut consumer).await.unwrap().expect("first decoded frame"); + let second = next_frame(&mut consumer).await.unwrap().expect("second decoded frame"); let first_frames = first.data.len() / size_of::(); let expected = advance(first.timestamp, first_frames, 48_000).unwrap(); assert_eq!(second.timestamp, expected); @@ -1806,9 +1970,9 @@ mod tests { write(&mut producer, 2, encoder.encode(&pcm).unwrap().payload); producer.finish().unwrap(); - let first = consumer.read().await.unwrap().expect("first decoded frame"); - let _second = consumer.read().await.unwrap().expect("second decoded frame"); - let resumed = consumer.read().await.unwrap().expect("resumed decoded frame"); + let first = next_frame(&mut consumer).await.unwrap().expect("first decoded frame"); + let _second = next_frame(&mut consumer).await.unwrap().expect("second decoded frame"); + let resumed = next_frame(&mut consumer).await.unwrap().expect("resumed decoded frame"); let first_frames = first.data.len() / size_of::(); let resumed_frames = resumed.data.len() / size_of::(); assert!(first_frames < frame_size, "the first epoch trims pre-skip"); @@ -1859,7 +2023,7 @@ mod tests { }) .unwrap(); - let frame = consumer.read().await.unwrap().expect("decoded frame"); + let frame = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); assert_eq!( Format::F32.as_interleaved_f32(&frame.data, 1).unwrap().as_ref(), samples @@ -1916,7 +2080,7 @@ mod tests { // The whole packet decodes: one 20 ms Opus frame at 48 kHz, less the pre-skip // trimmed off the first packet. Reading the fragment as legacy hands the codec // a slice of the moof instead, which still decodes, just to a shorter buffer. - let frame = consumer.read().await.unwrap().expect("decoded frame"); + let frame = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); // `as_micros`, not `==`: the CMAF path carries the fmp4 timescale and // `Timestamp`'s equality is structural, so the scales would have to match too. assert_eq!(frame.timestamp.as_micros(), 0); diff --git a/rs/moq-audio/src/decode/mod.rs b/rs/moq-audio/src/decode/mod.rs index 6f4500054c4..dc0850838ff 100644 --- a/rs/moq-audio/src/decode/mod.rs +++ b/rs/moq-audio/src/decode/mod.rs @@ -5,7 +5,7 @@ //! //! Entry points, high to low level: //! - [`Consumer`] subscribes to a track and hands back decoded -//! [`Frame`](crate::Frame)s. +//! [`Read`] events. //! - [`Decoder`] decodes packets you supply (bring your own payloads) into //! [`Decoded`] interleaved `f32` samples. //! @@ -19,6 +19,6 @@ mod consumer; mod decoded; mod decoder; -pub use consumer::{Consumer, Options, Output, Start}; +pub use consumer::{Consumer, Options, Output, Read, Start}; pub use decoded::Decoded; pub use decoder::{Config, Decoder, Kind}; diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index 85465c01af4..4f3650f31c3 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -559,6 +559,16 @@ impl Producer { #[cfg(test)] mod tests { + async fn decoded_frame(consumer: &mut crate::decode::Consumer) -> Result, Error> { + loop { + match consumer.read().await? { + Some(crate::decode::Read::Frame { frame, .. }) => return Ok(Some(frame)), + Some(crate::decode::Read::End { .. }) => {} + None => return Ok(None), + } + } + } + use std::time::Duration; use super::*; @@ -661,7 +671,7 @@ mod tests { producer.finish().unwrap(); let mut decoded = Vec::new(); - while let Some(frame) = audio.read().await.unwrap() { + while let Some(frame) = decoded_frame(&mut audio).await.unwrap() { let pcm = Format::F32.as_interleaved_f32(&frame.data, 1).unwrap(); decoded.extend_from_slice(&pcm); } @@ -834,7 +844,10 @@ mod tests { .unwrap(); producer.write(&full_frame(0)).unwrap(); - let first = audio.read().await.unwrap().expect("first packet before the reanchor"); + let first = decoded_frame(&mut audio) + .await + .unwrap() + .expect("first packet before the reanchor"); assert_eq!(first.data.len() / size_of::(), 960 - 312); producer.reanchor(); @@ -842,7 +855,7 @@ mod tests { producer.finish().unwrap(); let mut resumed_frames = 0; - while let Some(frame) = audio.read().await.unwrap() { + while let Some(frame) = decoded_frame(&mut audio).await.unwrap() { if frame.timestamp.as_micros() >= 1_000_000 { resumed_frames += frame.data.len() / size_of::(); } @@ -883,7 +896,7 @@ mod tests { let mut entered_dtx = false; for index in 0..100 { producer.write(&pcm_frame(&silence, index * 20_000)).unwrap(); - let consumed = consumer.read().await.unwrap().expect("one decoded frame"); + let consumed = decoded_frame(&mut consumer).await.unwrap().expect("one decoded frame"); assert_eq!(producer.activity(), consumed.activity); if consumed.activity.is_dtx() { entered_dtx = true; @@ -899,7 +912,7 @@ mod tests { }) .collect(); producer.write(&pcm_frame(&active, 2_000_000)).unwrap(); - let consumed = consumer.read().await.unwrap().expect("one decoded frame"); + let consumed = decoded_frame(&mut consumer).await.unwrap().expect("one decoded frame"); assert_eq!(producer.activity(), Activity::Active); assert_eq!(consumed.activity, Activity::Active); } diff --git a/rs/moq-audio/src/frame.rs b/rs/moq-audio/src/frame.rs index 2547b4a9b25..200641a97ee 100644 --- a/rs/moq-audio/src/frame.rs +++ b/rs/moq-audio/src/frame.rs @@ -5,7 +5,7 @@ use crate::Activity; /// One unit of raw PCM crossing the codec boundary: what /// [`encode::Producer::write`](crate::encode::Producer::write) takes and what -/// [`decode::Consumer::read`](crate::decode::Consumer::read) returns. +/// [`decode::Read::Frame`](crate::decode::Read::Frame) carries. /// /// Just a payload, a presentation timestamp, and whether the packet these /// samples came from coded any audio. PCM layout (format / sample rate / channel count) diff --git a/rs/moq-audio/src/playback.rs b/rs/moq-audio/src/playback.rs index 798bb5984dc..9277cf455ed 100644 --- a/rs/moq-audio/src/playback.rs +++ b/rs/moq-audio/src/playback.rs @@ -17,7 +17,8 @@ //! //! let mut sink = engine.sink(input)?; //! -//! while let Some(frame) = audio.read().await? { +//! while let Some(event) = audio.read().await? { +//! let moq_audio::decode::Read::Frame { frame, .. } = event else { continue }; //! let write = sink.write(&frame.data)?; //! if write.dropped_sample_frames > 0 { //! eprintln!("dropped {} live audio frames", write.dropped_sample_frames); diff --git a/rs/moq-audio/tests/integration/import.rs b/rs/moq-audio/tests/integration/import.rs index be750fb327e..86c217fb3fc 100644 --- a/rs/moq-audio/tests/integration/import.rs +++ b/rs/moq-audio/tests/integration/import.rs @@ -34,7 +34,7 @@ async fn ts_surround_opus_decodes() { // Canonical 5.1 order: FL, FR, FC, LFE, SL, SR. let mut energy = [0.0f64; 6]; let mut frames = 0; - while let Some(frame) = tokio::time::timeout(Duration::from_secs(5), consumer.read()) + while let Some(frame) = tokio::time::timeout(Duration::from_secs(5), next_frame(&mut consumer)) .await .expect("decoded frame timed out") .unwrap() @@ -62,3 +62,13 @@ async fn ts_surround_opus_decodes() { ); } } + +async fn next_frame(consumer: &mut decode::Consumer) -> Result, moq_audio::Error> { + loop { + match consumer.read().await? { + Some(decode::Read::Frame { frame, .. }) => return Ok(Some(frame)), + Some(decode::Read::End { .. }) => {} + None => return Ok(None), + } + } +} diff --git a/rs/moq-audio/tests/integration/roundtrip.rs b/rs/moq-audio/tests/integration/roundtrip.rs index 34ba2750638..3304c4b3e5a 100644 --- a/rs/moq-audio/tests/integration/roundtrip.rs +++ b/rs/moq-audio/tests/integration/roundtrip.rs @@ -70,7 +70,7 @@ async fn opus_round_trip_48k_stereo() { let mut total_frames = 0u64; let mut total_energy = 0.0f64; - while let Some(frame) = tokio::time::timeout(Duration::from_secs(5), consumer.read()) + while let Some(frame) = tokio::time::timeout(Duration::from_secs(5), next_frame(&mut consumer)) .await .expect("decoded frame timed out") .unwrap() @@ -146,7 +146,7 @@ async fn opus_round_trip_44100_s16_resampled() { producer.finish().unwrap(); let mut total_bytes = 0u64; - while let Some(frame) = tokio::time::timeout(Duration::from_secs(5), consumer.read()) + while let Some(frame) = tokio::time::timeout(Duration::from_secs(5), next_frame(&mut consumer)) .await .expect("decoded frame timed out") .unwrap() @@ -191,7 +191,7 @@ async fn pcm_round_trip_is_lossless() { .unwrap(); producer.finish().unwrap(); - let frame = consumer.read().await.unwrap().unwrap(); + let frame = next_frame(&mut consumer).await.unwrap().unwrap(); assert_eq!(frame.timestamp, Timestamp::from_micros(123_000).unwrap()); let decoded: Vec = frame .data @@ -201,7 +201,7 @@ async fn pcm_round_trip_is_lossless() { .map(|sample| f32::from_le_bytes([sample[0], sample[1], sample[2], sample[3]])) .collect(); assert_eq!(decoded, samples); - assert!(consumer.read().await.unwrap().is_none()); + assert!(next_frame(&mut consumer).await.unwrap().is_none()); } /// A 5.1 broadcast carries only its channel count in the catalog, which reads @@ -257,7 +257,7 @@ async fn pcm_five_one_reads_back_at_any_layout() { for (mut consumer, (_, layout, frame)) in consumers.into_iter().zip(cases) { assert_eq!(consumer.layout(), layout); - let decoded = consumer.read().await.unwrap().unwrap(); + let decoded = next_frame(&mut consumer).await.unwrap().unwrap(); let samples: Vec = decoded .data .as_chunks::<4>() @@ -271,3 +271,13 @@ async fn pcm_five_one_reads_back_at_any_layout() { } } } + +async fn next_frame(consumer: &mut decode::Consumer) -> Result, moq_audio::Error> { + loop { + match consumer.read().await? { + Some(decode::Read::Frame { frame, .. }) => return Ok(Some(frame)), + Some(decode::Read::End { .. }) => {} + None => return Ok(None), + } + } +} diff --git a/rs/moq-c/src/audio.rs b/rs/moq-c/src/audio.rs index be2ef17ba0c..58b3488ad43 100644 --- a/rs/moq-c/src/audio.rs +++ b/rs/moq-c/src/audio.rs @@ -259,7 +259,8 @@ impl Audio { biased; _ = &mut close => return Ok(()), frame = consumer.read() => match frame { - Ok(Some(frame)) => frame, + Ok(Some(moq_audio::decode::Read::Frame { frame, .. })) => frame, + Ok(Some(moq_audio::decode::Read::End { .. })) => continue, Ok(None) => return Ok(()), // One packet the codec rejected is that packet's problem: the // decoder stays usable, so drop it and keep the subscription rather diff --git a/rs/moq-cli/src/play/buffer.rs b/rs/moq-cli/src/play/buffer.rs index 9c458574873..66a92e52177 100644 --- a/rs/moq-cli/src/play/buffer.rs +++ b/rs/moq-cli/src/play/buffer.rs @@ -18,6 +18,8 @@ pub(super) struct Buffer { pub floor: Option, waiting_keyframe: bool, pub ended: bool, + /// Exclusive endpoint declared for this encoder segment. + pub end: Option, } impl Default for Buffer { @@ -32,6 +34,7 @@ impl Default for Buffer { // A join resumes at its first keyframe, like any other break. waiting_keyframe: true, ended: false, + end: None, } } } diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 96c2f7c41fd..d0a19b62c85 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -16,10 +16,18 @@ use tokio::time::Instant; use super::args::Args; use super::output::{Output, Sink, Speaker}; use super::playback::{Kind, Playback, Track, joined}; -use super::timeline::{AudioTimeline, Presentation, fit, timestamp}; +use super::timeline::{AudioTimeline, Presentation, Role, Transition, fit, timestamp}; use super::video::Video; use super::window::Event; +/// What a frame that starts a new decoded timeline crossed. +enum Crossing { + /// A declared end, whose transition the frame resumes. + Resume(Transition), + /// Lost media: no end was declared. + Loss, +} + /// The floor on the speaker's buffer, whatever delay was asked for. The device /// pulls on a fixed clock, so a ring shallower than this drops out on ordinary /// network jitter, and one with no depth at all can never be read from. @@ -419,10 +427,16 @@ async fn play_audio( // The playout delay is the consumer's to size, from how unevenly packets arrive, // and the sink is where it lives: a sample handed over now sounds that much // later. That estimate times each packet when `read` hands it over, so this - // loop never waits on the speaker between reads. It writes everything as it - // comes and lets `fit` hold the sink on the target instead. The window + // receiver never waits on the speaker between reads. Live playback lets + // `fit` hold the sink on the target; declared ends queue their replacements + // until the old sink drains. The window // schedules video against where the speaker actually is, which keeps the two // together whatever the target does. + let role = Role::new(presentation.clone(), Kind::Audio); + let received = Mutex::new(VecDeque::new()); + let arrived = tokio::sync::Notify::new(); + let popped = tokio::sync::Notify::new(); + let max_delay = consumer.max_delay(); let sample_rate = consumer.sample_rate(); let layout = consumer.layout(); let channels = layout.channels(); @@ -449,88 +463,239 @@ async fn play_audio( // decoder's latency budget says: anything longer is what that budget chose to // skip, so playing it as silence would hand back the delay the skip avoided. // Past it the sink skips the hole and the clock re-anchors, as it does today. - let fill_max = samples(consumer.max_delay()); + let fill_max = samples(max_delay); let mut timeline = AudioTimeline::default(); + // How far ahead of its due time a frame is written: the playout target less + // the frame itself, never under the sink's floor. + let lead = |delay: Duration, frame: &moq_audio::Frame| { + let duration = Duration::from_secs_f64((frame.data.len() / stride) as f64 / sample_rate as f64); + latency.max(delay.min(max_delay).max(AUDIO_BUFFER_MIN).saturating_sub(duration)) + }; + // Tracks whether the last read failed, so a stream the decoder can't read at // all logs once rather than once per packet. let mut dropping = false; - loop { - let frame = match consumer.read().await { - Ok(Some(frame)) => frame, - Ok(None) => break, - // One bad packet is that packet's problem: the decoder stays usable, so - // skip it rather than ending playback and taking the video window down - // with it. - Err(err @ moq_audio::Error::Decode(_)) => { - if dropping { - tracing::debug!(%err, "dropping an audio frame"); - } else { - tracing::warn!(%err, "dropping an audio frame"); - dropping = true; + // Resolved in delivery order, ahead of playback, so each end's transition is + // declared before the replacement media that resumes it is planned. A short + // timeline can declare its end before its first PCM, so ends queue until a + // frame crosses them. + let receive = async { + let mut current = None; + let mut ends = VecDeque::new(); + loop { + // Stop reading while play is held behind a resume with more than the age + // budget queued, so the subscription evicts the excess instead of this + // queue growing without bound. + while received + .lock() + .unwrap() + .front() + .is_some_and(|(.., queued): &(_, _, _, Instant)| queued.elapsed() > max_delay) + { + popped.notified().await; + } + let event = match consumer.read().await { + Err(err) if !matches!(err, moq_audio::Error::Decode(_)) => return Err(err.into()), + event => event, + }; + let ended = matches!(event, Ok(None)); + let delay = consumer.delay(); + let crossing = match &event { + Ok(Some(moq_audio::decode::Read::End { discontinuity, .. })) => { + ends.push_back((*discontinuity, role.declare())); + None } - continue; + Ok(Some(moq_audio::decode::Read::Frame { discontinuity, frame })) + if current != Some(*discontinuity) => + { + // The first frame continues nothing, unless a track joined while paused + // declared an end before it. + let first = current.replace(*discontinuity).is_none(); + // Every end before this timeline was crossed. Only the last one has + // replacement media; the ones before it ended empty timelines. + let mut crossed = None; + while let Some(&(ended, transition)) = ends.front() + && ended < *discontinuity + { + ends.pop_front(); + if let Some(skipped) = crossed.replace(transition) { + role.abandon(skipped); + } + } + match crossed { + Some(transition) => { + role.plan(transition, frame.timestamp, Some(lead(delay, frame))); + Some(Crossing::Resume(transition)) + } + None if first => None, + None => Some(Crossing::Loss), + } + } + _ => None, + }; + received + .lock() + .unwrap() + .push_back((event, delay, crossing, Instant::now())); + arrived.notify_one(); + if ended { + return Ok::<_, anyhow::Error>(()); } - Err(err) => return Err(err.into()), - }; - dropping = false; - - let length = frame.data.len() / stride; - let start = timestamp(frame.timestamp); - let timing = timeline.push(start, length, sample_rate, fill_max); - - // A rewind or a hole too large to fill starts a new playback sink. The old - // sink has no media clock, so its buffered audio cannot be carried across a - // timeline region the player skipped. - if timing.reset_sink { - drop(sink); - presentation.lock().unwrap().restarted(); - sink = speaker.sink(input.clone())?; - dry = true; } + }; + let play = async { + // The timeline the role's mapping serves, and an end declared for a later + // one, which can only be mapped once that timeline resumes. + let mut mapped = None; + let mut deferred = None; + let mut sequential = false; + loop { + let (event, delay, crossing, _) = loop { + if let Some(event) = received.lock().unwrap().pop_front() { + popped.notify_one(); + break event; + } + arrived.notified().await; + }; + let frame = match event { + Ok(Some(moq_audio::decode::Read::Frame { discontinuity, frame })) => { + mapped = Some(discontinuity); + match crossing { + Some(Crossing::Loss) => { + deferred = None; + role.gap(); + sequential = false; + } + Some(Crossing::Resume(transition)) => { + // A declared boundary preserves the old sink, unlike an undeclared hole. + drain(sink).await; + let first = lead(delay, &frame); + let mapping = role.resume(transition, frame.timestamp, Some(first)).await?; + role.apply(transition, mapping); + let at = presentation + .lock() + .unwrap() + .due(role.map(frame.timestamp)?) + .and_then(|at| at.checked_sub(first)); + if let Some(at) = at { + tokio::time::sleep_until(at.into()).await; + } + if let Some((ended, end)) = deferred.take() + && ended == discontinuity + { + role.end(end)?; + } + sequential = true; + timeline = AudioTimeline::default(); + sink = speaker.sink(input.clone())?; + dry = true; + } + None => {} + } + frame + } + Ok(Some(moq_audio::decode::Read::End { discontinuity, end })) => { + if mapped.is_none_or(|mapped| mapped == discontinuity) { + role.end(end)?; + } else { + deferred = Some((discontinuity, end)); + } + continue; + } + Ok(None) => break, + // One bad packet is that packet's problem: the decoder stays usable, so + // skip it rather than ending playback and taking the video window down + // with it. + Err(err @ moq_audio::Error::Decode(_)) => { + if dropping { + tracing::debug!(%err, "dropping an audio frame"); + } else { + tracing::warn!(%err, "dropping an audio frame"); + dropping = true; + } + continue; + } + Err(err) => return Err(err.into()), + }; + dropping = false; + + let length = frame.data.len() / stride; + let mapped = role.map(frame.timestamp)?; + let start = timestamp(mapped); + let timing = timeline.push(start, length, sample_rate, fill_max); + + // A rewind or a hole too large to fill starts a new playback sink. The old + // sink has no media clock, so its buffered audio cannot be carried across a + // timeline region the player skipped. + if timing.reset_sink { + sequential = false; + drop(sink); + presentation.lock().unwrap().restarted(); + sink = speaker.sink(input.clone())?; + dry = true; + } - // A hole in the media is a hole in the audio, not a splice. Handing the next - // frame straight to the speaker shortens the track by the missing duration, - // which leaves it running ahead of media time until the clock below - // re-anchors, taking the video with it. So the hole goes in as silence ahead - // of the frame, and the pair is fitted to the target as one write. - let buffered = samples(sink.buffered()); - dry |= buffered == 0; - // Capped at the age budget: audio older than it is skipped rather than held, - // so a deeper target could never fill. The advertised floor needs the cap, - // being a number the publisher declared about itself, unbounded. The budget - // is also what the sink's ring was sized to hold. - let target = samples(consumer.delay().min(consumer.max_delay()).max(AUDIO_BUFFER_MIN)); - let fit = fit(dry, buffered, target, slack, timing.silence + length as u64); - dry = false; - - // Playback drops stay on the live timeline; retrying them would add latency, - // and the sink already reports them in its logs. - let mut quiet = fit.pad + timing.silence.saturating_sub(fit.skip); - while quiet > 0 { - let part = (quiet as usize * stride).min(silence.len()); - sink.write(&silence[..part])?; - quiet -= (part / stride) as u64; - } - let skip = fit.skip.saturating_sub(timing.silence) as usize * stride; - sink.write(&frame.data[skip.min(frame.data.len())..])?; - - // Anchor the playout clock on where the speaker has actually reached, which - // is the only half of the pipeline that cannot skip ahead. A move has to - // wake the window: it is asleep on a deadline computed from the old anchor, - // and every queued frame is due earlier now. - let moved = presentation - .lock() - .unwrap() - .audio(timing.end, sink.buffered(), Instant::now().into_std()); - if moved { - changed.notify_one(); - output.send(Event::Wake); + // Resumed media was received while the old tail played. Keep its + // spacing instead of fitting that queued burst as if it arrived live. + // The lead follows the current target so jitter can still deepen the sink. + if sequential { + let at = presentation + .lock() + .unwrap() + .due(mapped) + .and_then(|at| at.checked_sub(lead(delay, &frame))); + if let Some(at) = at { + tokio::time::sleep_until(at.into()).await; + } + } + + // A hole in the media is a hole in the audio, not a splice. Handing the next + // frame straight to the speaker shortens the track by the missing duration, + // which leaves it running ahead of media time until the clock below + // re-anchors, taking the video with it. So the hole goes in as silence ahead + // of the frame, and the pair is fitted to the target as one write. + let buffered = samples(sink.buffered()); + dry |= buffered == 0; + // Capped at the age budget: audio older than it is skipped rather than held, + // so a deeper target could never fill. The advertised floor needs the cap, + // being a number the publisher declared about itself, unbounded. The budget + // is also what the sink's ring was sized to hold. + let target = samples(delay.min(max_delay).max(AUDIO_BUFFER_MIN)); + let fit = fit(dry, buffered, target, slack, timing.silence + length as u64); + dry = false; + + // Playback drops stay on the live timeline; retrying them would add latency, + // and the sink already reports them in its logs. + let mut quiet = fit.pad + timing.silence.saturating_sub(fit.skip); + while quiet > 0 { + let part = (quiet as usize * stride).min(silence.len()); + sink.write(&silence[..part])?; + quiet -= (part / stride) as u64; + } + let skip = fit.skip.saturating_sub(timing.silence) as usize * stride; + sink.write(&frame.data[skip.min(frame.data.len())..])?; + + // Anchor the playout clock on where the speaker has actually reached, which + // is the only half of the pipeline that cannot skip ahead. A move has to + // wake the window: it is asleep on a deadline computed from the old anchor, + // and every queued frame is due earlier now. + let moved = presentation + .lock() + .unwrap() + .audio(timing.end, sink.buffered(), Instant::now().into_std()); + if moved { + changed.notify_one(); + output.send(Event::Wake); + } } - } + Ok::<_, anyhow::Error>(sink) + }; + let (_, sink) = tokio::try_join!(receive, play)?; + role.retire(Instant::now().into_std().checked_add(sink.buffered())); Ok(sink) } @@ -557,6 +722,516 @@ mod tests { const PACKET: u64 = 960; const PACKET_DURATION: Duration = Duration::from_millis(20); + #[tokio::test] + async fn a_quick_audio_resume_preserves_the_old_speaker_tail() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Audio)); + let mut options = moq_audio::decode::Options::new(); + options.delay = Some(delay); + options.max_delay = delay; + options.output.format = moq_audio::Format::F32; + let consumer = moq_audio::decode::Consumer::new( + &broadcast.consume(), + &AudioConfig::new(AudioCodec::Pcm, SAMPLE_RATE, 1), + "audio", + options, + ) + .await + .unwrap(); + let recorder = Recorder::default(); + let playback = AudioPlayback { + changed: Default::default(), + speaker: recorder.clone(), + presentation: Arc::new(Mutex::new(Presentation::new(delay))), + latency: delay, + output: recorder.clone(), + }; + let task = tokio::spawn(async move { + play_audio(consumer, playback).await.unwrap().finish().await; + }); + producer.write(packet(0, 0.25)).unwrap(); + tokio::task::yield_now().await; + producer.discontinuity().unwrap(); + producer.write(packet(0, 0.5)).unwrap(); + producer.finish().unwrap(); + task.await.unwrap(); + let played = recorder.played(); + let old: Vec<_> = played.iter().filter(|part| part.sample == 0.25).collect(); + let new: Vec<_> = played.iter().filter(|part| part.sample == 0.5).collect(); + assert_eq!( + old.iter().map(|part| part.to - part.from).sum::(), + PACKET_DURATION + ); + assert_eq!( + new.iter().map(|part| part.to - part.from).sum::(), + PACKET_DURATION + ); + assert!(old.last().unwrap().to <= new.first().unwrap().from); + } + + #[tokio::test] + async fn paired_quick_resumes_keep_audio_and_reordered_video_together() { + quick_resumes(true, true, Duration::from_millis(200)).await; + } + + #[tokio::test] + async fn an_audio_resume_aligns_with_continuous_video() { + quick_resumes(true, false, Duration::from_millis(200)).await; + } + + #[tokio::test] + async fn a_video_resume_aligns_with_continuous_audio() { + quick_resumes(false, true, Duration::from_millis(200)).await; + } + + #[tokio::test] + async fn paired_resumes_follow_the_speaker_when_delays_differ() { + quick_resumes(true, true, Duration::from_millis(80)).await; + } + + async fn quick_resumes(audio_end: bool, video_end: bool, audio_latency: Duration) { + struct Buffered(Vec); + impl super::super::video::Decoder for Buffered { + async fn decode(&mut self, frame: moq_mux::container::Frame) -> anyhow::Result> { + self.0.push(moq_video::Frame::new( + moq_video::Surface::rgba(&[128; 16 * 16 * 4], moq_video::Size::new(16, 16))?, + frame.timestamp, + )); + self.0.sort_by_key(|frame| frame.timestamp); + let ready = self.0.len().saturating_sub(2); + Ok(self.0.drain(..ready).collect()) + } + async fn flush(&mut self) -> anyhow::Result> { + Ok(std::mem::take(&mut self.0)) + } + } + tokio::time::pause(); + let playback_start = Instant::now(); + let delay = Duration::from_millis(200); + let recorder = Recorder::default(); + let origin = moq_tokio::origin::spawn(); + let media = media(&origin, delay, recorder.clone()); + let broadcast = moq_net::broadcast::Info::new().produce(); + let audio = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let video = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = video + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(delay)) + .await + .unwrap(); + let mut audio = moq_mux::container::Producer::new(audio, Container::Legacy(moq_mux::container::Kind::Audio)); + let mut video = moq_mux::container::Producer::new(video, Container::Legacy(moq_mux::container::Kind::Video)); + let mut options = moq_audio::decode::Options::new(); + options.delay = Some(delay); + options.max_delay = delay; + options.output.format = moq_audio::Format::F32; + let consumer = moq_audio::decode::Consumer::new( + &broadcast.consume(), + &AudioConfig::new(AudioCodec::Pcm, SAMPLE_RATE, 1), + "audio", + options, + ) + .await + .unwrap(); + let audio_play = AudioPlayback { + changed: media.drained.clone(), + speaker: recorder.clone(), + presentation: media.presentation.clone(), + latency: audio_latency, + output: recorder.clone(), + }; + let video_play = Video { + presentation: media.presentation.clone(), + frames: media.video.clone(), + changed: media.drained.clone(), + output: recorder.clone(), + max_delay: delay, + }; + let audio_task = tokio::spawn(async move { + play_audio(consumer, audio_play).await.unwrap().finish().await; + }); + let video_task = tokio::spawn(video_play.run( + moq_mux::container::Consumer::new(subscriber, Container::Legacy(moq_mux::container::Kind::Video)), + Buffered(Vec::new()), + )); + let picture = |index: u64| moq_mux::container::Frame { + timestamp: moq_net::Timestamp::from_millis(index * 20).unwrap(), + duration: Some(moq_net::Timestamp::from_millis(20).unwrap()), + keyframe: index == 0, + payload: Bytes::from_static(b"picture"), + }; + for index in 0..3 { + let mut pcm = packet(index, 0.25); + pcm.keyframe = index == 0; + audio.write(pcm).unwrap(); + video.write(picture(index)).unwrap(); + tokio::time::sleep(PACKET_DURATION).await; + } + if audio_end { + audio.discontinuity().unwrap(); + } + if video_end { + video.discontinuity().unwrap(); + } + for index in 0..3 { + if audio_end || index == 0 { + let mut pcm = packet(if audio_end { index } else { index + 3 }, 0.5); + pcm.keyframe = index == 0; + audio.write(pcm).unwrap(); + } + if video_end || index == 0 { + video.write(picture(if video_end { index } else { index + 3 })).unwrap(); + } + } + if audio_end { + audio.finish().unwrap(); + } + if video_end { + video.finish().unwrap(); + } + let mut shown = Vec::new(); + for step in 0..800 { + if step >= 20 && step % 20 == 0 { + let index = 3 + step / 20; + if !audio_end { + audio.write(packet(index, 0.5)).unwrap(); + } + if !video_end { + video.write(picture(index)).unwrap(); + } + } + tokio::task::yield_now().await; + if let Some(timestamp) = recorder.present(&media) { + shown.push((timestamp, Instant::now())); + } + tokio::time::advance(Duration::from_millis(1)).await; + } + if !audio_end { + audio.finish().unwrap(); + } + if !video_end { + video.finish().unwrap(); + } + audio_task.await.unwrap(); + video_task.await.unwrap().unwrap(); + if video_end { + assert_eq!(shown.len(), 6, "both decoder tails must be shown"); + } + assert_eq!( + shown[..3] + .iter() + .map(|(timestamp, _)| timestamp.as_millis()) + .collect::>(), + [0, 20, 40] + ); + let played = recorder.played(); + let old: Vec<_> = played.iter().filter(|part| part.sample == 0.25).collect(); + let new: Vec<_> = played.iter().filter(|part| part.sample == 0.5).collect(); + assert_eq!( + old.iter().map(|part| part.to - part.from).sum::(), + PACKET_DURATION * 3 + ); + if audio_end { + assert_eq!( + new.iter().map(|part| part.to - part.from).sum::(), + PACKET_DURATION * 3 + ); + } + assert!(old.last().unwrap().to <= new[0].from); + let (picture, sample) = if audio_end && video_end { + (shown[3].1, new[0].from) + } else if audio_end { + let timestamp = new[0].from.duration_since(playback_start + delay); + let shown = shown + .iter() + .find(|(shown, _)| { + let start = super::timestamp(*shown); + start <= timestamp && timestamp < start + PACKET_DURATION + }) + .expect("continuous video reaches resumed audio"); + (shown.1 + (timestamp - super::timestamp(shown.0)), new[0].from) + } else { + let timestamp = super::timestamp(shown[3].0); + let index = (shown[3].0.as_millis() / 20) as usize; + let offset = timestamp - PACKET_DURATION * index as u32; + (shown[3].1, new[index - 3].from + offset) + }; + assert!( + picture.saturating_duration_since(sample) <= Duration::from_millis(1) + && sample.saturating_duration_since(picture) <= Duration::from_millis(1), + "new A/V clocks diverged: picture={picture:?} sample={sample:?}" + ); + } + + #[tokio::test] + async fn an_end_before_its_first_pcm_still_resumes_that_timeline() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Audio)); + let mut options = moq_audio::decode::Options::new(); + options.delay = Some(delay); + options.max_delay = delay; + options.output.format = moq_audio::Format::F32; + options.output.sample_rate = Some(SAMPLE_RATE); + let consumer = moq_audio::decode::Consumer::new( + &broadcast.consume(), + &AudioConfig::new(AudioCodec::Pcm, 44_100, 1), + "audio", + options, + ) + .await + .unwrap(); + let recorder = Recorder::default(); + let playback = AudioPlayback { + changed: Default::default(), + speaker: recorder.clone(), + presentation: Arc::new(Mutex::new(Presentation::new(delay))), + latency: delay, + output: recorder.clone(), + }; + let task = tokio::spawn(async move { + play_audio(consumer, playback).await.unwrap().finish().await; + }); + let pcm = |micros: u64, samples: usize, sample: f32| moq_mux::container::Frame { + timestamp: moq_net::Timestamp::from_micros(micros).unwrap(), + duration: Some(moq_net::Timestamp::from_scale(samples as u64, 44_100).unwrap()), + keyframe: true, + payload: std::iter::repeat_n(sample.to_le_bytes(), samples) + .flatten() + .collect::>() + .into(), + }; + for index in 0..4 { + producer.write(pcm(index * 23_220, 1024, 0.25)).unwrap(); + } + producer.discontinuity().unwrap(); + // Too short for the resampler to emit anything before this encoder ends too. + producer.write(pcm(100_000, 64, 0.5)).unwrap(); + producer.discontinuity().unwrap(); + for index in 0..4 { + producer.write(pcm(200_000 + index * 23_220, 1024, 0.75)).unwrap(); + } + producer.finish().unwrap(); + task.await.unwrap(); + let played = recorder.played(); + let sinks = |range: std::ops::Range| { + played + .iter() + .filter(|part| range.contains(&part.sample)) + .map(|part| part.sink) + .collect::>() + }; + let (old, short, new) = (sinks(0.2..0.3), sinks(0.4..0.6), sinks(0.7..0.8)); + assert!(!short.is_empty(), "the short timeline played: {played:?}"); + assert!( + old.is_disjoint(&short), + "the short timeline resumed on its own sink: {played:?}" + ); + assert!( + short.is_disjoint(&new), + "the next timeline resumed on its own sink: {played:?}" + ); + } + + #[tokio::test] + async fn joining_a_paused_track_resumes_on_its_first_frame() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Audio)); + for index in 0..3 { + producer.write(packet(index, 0.25)).unwrap(); + } + producer.discontinuity().unwrap(); + // Joins at the end marker: the stopped encoder's media is already behind it. + let mut options = moq_audio::decode::Options::new(); + options.delay = Some(delay); + options.max_delay = delay; + options.start = moq_audio::decode::Start::Latest; + options.output.format = moq_audio::Format::F32; + let consumer = moq_audio::decode::Consumer::new( + &broadcast.consume(), + &AudioConfig::new(AudioCodec::Pcm, SAMPLE_RATE, 1), + "audio", + options, + ) + .await + .unwrap(); + let recorder = Recorder::default(); + let playback = AudioPlayback { + changed: Default::default(), + speaker: recorder.clone(), + presentation: Arc::new(Mutex::new(Presentation::new(delay))), + latency: delay, + output: recorder.clone(), + }; + let task = tokio::spawn(async move { + play_audio(consumer, playback).await.unwrap().finish().await; + }); + tokio::task::yield_now().await; + for index in 3..6 { + producer.write(packet(index, 0.5)).unwrap(); + } + producer.finish().unwrap(); + task.await.unwrap(); + let played = recorder.played(); + let resumed: Vec<_> = played.iter().filter(|part| part.sample == 0.5).collect(); + assert!(!resumed.is_empty(), "{played:?}"); + assert!( + resumed.iter().all(|part| part.sink > 0), + "the replacement resumed its declared transition: {played:?}" + ); + } + + #[tokio::test] + async fn a_held_audio_resume_leaves_the_excess_to_the_subscription() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let mut producer = moq_mux::container::Producer::new(track, Container::Legacy(moq_mux::container::Kind::Audio)); + let mut options = moq_audio::decode::Options::new(); + options.delay = Some(delay); + options.max_delay = delay; + options.output.format = moq_audio::Format::F32; + let consumer = moq_audio::decode::Consumer::new( + &broadcast.consume(), + &AudioConfig::new(AudioCodec::Pcm, SAMPLE_RATE, 1), + "audio", + options, + ) + .await + .unwrap(); + let recorder = Recorder::default(); + let presentation = Arc::new(Mutex::new(Presentation::new(delay))); + let changed = Arc::new(tokio::sync::Notify::new()); + let playback = AudioPlayback { + changed: changed.clone(), + speaker: recorder.clone(), + presentation: presentation.clone(), + latency: delay, + output: recorder.clone(), + }; + let task = tokio::spawn(async move { + play_audio(consumer, playback).await.unwrap().finish().await; + }); + for index in 0..5 { + producer.write(packet(index, 0.25)).unwrap(); + tokio::time::sleep(PACKET_DURATION).await; + } + // Video declares an end two seconds out, holding the audio replacement back. + let video = Role::new(presentation.clone(), Kind::Video); + video.declare(); + video.end(moq_net::Timestamp::from_millis(2_000).unwrap()).unwrap(); + producer.discontinuity().unwrap(); + const REPLACEMENT: u64 = 100; + for index in 0..REPLACEMENT { + producer.write(packet(5 + index, 0.5)).unwrap(); + tokio::time::sleep(PACKET_DURATION).await; + } + producer.finish().unwrap(); + task.await.unwrap(); + drop(video); + let (resumed, _) = played(&recorder, 0.5); + assert!(!resumed.is_zero(), "the replacement played"); + assert!( + resumed < PACKET_DURATION * REPLACEMENT as u32 / 2, + "media held behind the resume past the age budget was queued, not evicted: {resumed:?}" + ); + } + + #[tokio::test] + async fn an_evicted_replacement_gop_gives_way_to_the_next_replacement() { + struct Passthrough; + impl super::super::video::Decoder for Passthrough { + async fn decode(&mut self, frame: moq_mux::container::Frame) -> anyhow::Result> { + Ok(vec![moq_video::Frame::new( + moq_video::Surface::rgba(&[128; 16 * 16 * 4], moq_video::Size::new(16, 16))?, + frame.timestamp, + )]) + } + async fn flush(&mut self) -> anyhow::Result> { + Ok(Vec::new()) + } + } + tokio::time::pause(); + let delay = Duration::from_millis(200); + let recorder = Recorder::default(); + let origin = moq_tokio::origin::spawn(); + let media = media(&origin, delay, recorder.clone()); + let broadcast = moq_net::broadcast::Info::new().produce(); + let video = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + // The network keeps every group; only the player's age budget evicts. + let subscriber = video + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))) + .await + .unwrap(); + let mut video = moq_mux::container::Producer::new(video, Container::Legacy(moq_mux::container::Kind::Video)); + let video_play = Video { + presentation: media.presentation.clone(), + frames: media.video.clone(), + changed: media.drained.clone(), + output: recorder.clone(), + max_delay: delay, + }; + let video_task = tokio::spawn(video_play.run( + moq_mux::container::Consumer::new(subscriber, Container::Legacy(moq_mux::container::Kind::Video)), + Passthrough, + )); + let picture = |millis: u64, keyframe: bool| moq_mux::container::Frame { + timestamp: moq_net::Timestamp::from_millis(millis).unwrap(), + duration: Some(moq_net::Timestamp::from_millis(20).unwrap()), + keyframe, + payload: Bytes::from_static(b"picture"), + }; + for index in 0..3 { + video.write(picture(index * 20, index == 0)).unwrap(); + tokio::time::sleep(PACKET_DURATION).await; + } + video.discontinuity().unwrap(); + // A burst longer than the age budget evicts this replacement's only keyframe. + for index in 0..15 { + video.write(picture(index * 20, index == 0)).unwrap(); + } + video.discontinuity().unwrap(); + for index in 0..3 { + video.write(picture(5_000 + index * 20, index == 0)).unwrap(); + } + video.finish().unwrap(); + let mut shown = Vec::new(); + for _ in 0..2_000 { + tokio::task::yield_now().await; + if let Some(timestamp) = recorder.present(&media) { + shown.push(timestamp.as_millis()); + } + tokio::time::advance(Duration::from_millis(1)).await; + } + video_task.await.unwrap().unwrap(); + assert_eq!(shown[..3], [0, 20, 40], "{shown:?}"); + assert_eq!(shown.len(), 6, "the next replacement plays: {shown:?}"); + } + /// A mono PCM rendition, published and named in the catalog until dropped. fn rendition( broadcast: &moq_net::broadcast::Producer, diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index 53f5e41ece0..d4e8d330e55 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -1,7 +1,9 @@ //! Media time: the playout clock, and the audio track's own timeline. +use anyhow::Context; use std::time::{Duration, Instant}; +use super::playback::Kind; use hang::moq_net::Timestamp; /// The playout clock: one anchor the window and the speaker both present @@ -28,6 +30,49 @@ pub(super) struct Presentation { /// audio frame pins the anchor outright instead of pacing against media it /// will never play. restart: bool, + /// Endpoints declared by each live role, on the shared playback timeline. + ends: [Option; 2], + /// When each retired role's tail finishes playing, frozen on the wall clock: + /// a later rendition re-pins the anchor, which would carry a media endpoint + /// onto an unrelated timeline. Kept apart from `ends` so a successor role + /// never clears its predecessor's tail. + tails: Vec, + /// Encoder restarts whose replacements have not all started playing. + transitions: std::collections::VecDeque, + /// The latest transition each role declared. + declared: [Transition; 2], + next: Transition, + reference: Option, + /// Wakes every pending resume when a boundary, a transition or the anchor + /// changes. Kept apart from the window's and the decoders' wakeups, so a + /// resume can never take a wakeup meant for them. + timeline: std::sync::Arc, +} + +/// One encoder restart, shared by the roles whose ends declared it. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] +pub(super) struct Transition(u64); + +struct Pending { + id: Transition, + /// Roles whose declared end opened or joined this transition. + members: [bool; 2], + /// Members that applied its mapping. + resumed: [bool; 2], + /// Each member's first replacement media. + plans: [Option; 2], + mapping: Option, +} + +struct Boundary { + starts: Option, + end: Duration, +} + +#[derive(Clone, Copy)] +struct Plan { + source: Duration, + delay: Duration, } impl Presentation { @@ -37,9 +82,49 @@ impl Presentation { delay, speaker: false, restart: false, + ends: Default::default(), + tails: Vec::new(), + transitions: Default::default(), + declared: Default::default(), + next: Transition(1), + reference: None, + timeline: Default::default(), } } + /// How many transitions are still pending. + #[cfg(test)] + pub(super) fn pending(&self) -> usize { + self.transitions.len() + } + + fn transition(&mut self, id: Transition) -> Option<&mut Pending> { + self.transitions.iter_mut().find(|pending| pending.id == id) + } + + /// Forget transitions nobody resumes, or that every member resumed once + /// their replacement started and nothing can join them. + fn prune(&mut self, now: Instant) { + self.transitions.retain(|pending| { + let members = pending.members.iter().any(|member| *member); + let resumed = (0..2).all(|role| !pending.members[role] || pending.resumed[role]); + let started = pending.mapping.is_some_and(|mapping| mapping.started(now)); + members && !(resumed && started) + }); + } + + /// The media position the shared clock will reach at `at`. + fn position(&self, at: Instant) -> Option { + let reference = self.reference?; + let due = self.due(reference)?; + let reference = timestamp(reference); + Some(if at >= due { + reference.saturating_add(at.duration_since(due)) + } else { + reference.saturating_sub(due.duration_since(at)) + }) + } + /// Fold an arriving video frame into the anchor, unless the speaker owns it, /// reporting whether that moved the schedule. /// @@ -50,14 +135,20 @@ impl Presentation { return false; } + self.reference = Some(timestamp); let before = self.pacer.due(timestamp); - before != Some(self.pacer.pace(timestamp, now)) + let moved = before != Some(self.pacer.pace(timestamp, now)); + if moved { + self.timeline.notify_waiters(); + } + moved } /// Start a new video timeline without taking ownership from a speaker. pub(super) fn video_restarted(&mut self) { if !self.speaker { self.pacer = moq_mux::Pacer::default().with_delay(self.delay); + self.timeline.notify_waiters(); } } @@ -75,6 +166,7 @@ impl Presentation { /// recompute the deadline it is already asleep on. pub(super) fn audio(&mut self, end: Duration, buffered: Duration, now: Instant) -> bool { let timestamp = media(end); + self.reference = Some(timestamp); let edge = now .checked_add(buffered) .and_then(|at| at.checked_sub(self.delay)) @@ -93,7 +185,11 @@ impl Presentation { }; self.speaker = true; - before != Some(after) + let moved = before != Some(after); + if moved { + self.timeline.notify_waiters(); + } + moved } /// The speaker restarted on a new timeline, so the media either side of the @@ -123,6 +219,262 @@ impl Presentation { } } +/// Maps each encoder segment onto the shared playback clock. +pub(super) struct Role { + presentation: std::sync::Arc>, + timeline: std::sync::Arc, + role: Kind, + mapping: std::sync::Mutex, +} + +#[derive(Clone, Copy, Default)] +pub(super) struct Mapping { + source: Duration, + target: Duration, + starts: Option, +} + +impl Mapping { + fn map(self, source: Timestamp) -> anyhow::Result { + let source = Duration::from(source); + let target = if source >= self.source { + self.target + .checked_add(source - self.source) + .context("playback timeline overflow")? + } else { + self.target + .checked_sub(self.source - source) + .context("picture precedes the playback timeline")? + }; + Ok(Timestamp::from_micros(u64::try_from(target.as_micros())?)?) + } + + fn started(&self, now: Instant) -> bool { + self.starts.is_none_or(|starts| starts <= now) + } +} + +impl Role { + pub(super) fn new(presentation: std::sync::Arc>, role: Kind) -> Self { + let timeline = presentation.lock().unwrap().timeline.clone(); + Self { + presentation, + timeline, + role, + mapping: Default::default(), + } + } + + /// Stop while a tail still plays until `drained`, holding a declared end until + /// then. Dropping instead discards the tail with the end. + pub(super) fn retire(self, drained: Option) { + let now = tokio::time::Instant::now().into_std(); + let mut presentation = self.presentation.lock().unwrap(); + presentation.tails.retain(|tail| *tail > now); + if presentation.ends[self.role as usize].take().is_some() + && let Some(drained) = drained + { + presentation.tails.push(drained); + } + } + + pub(super) fn map(&self, source: Timestamp) -> anyhow::Result { + self.mapping.lock().unwrap().map(source) + } + + /// Declare that this role's encoder stopped, returning the transition its + /// replacement resumes. + /// + /// A peer's transition this role has not joined yet is the same restart, as + /// long as its replacement has not started playing; otherwise this one opens + /// a transition of its own. + pub(super) fn declare(&self) -> Transition { + let now = tokio::time::Instant::now().into_std(); + let own = self.role as usize; + let mut presentation = self.presentation.lock().unwrap(); + presentation.prune(now); + let after = presentation.declared[own]; + let joined = presentation + .transitions + .iter_mut() + .filter(|transition| transition.id > after) + .find(|transition| { + transition.members[1 - own] + && !transition.members[own] + && transition.mapping.is_none_or(|mapping| !mapping.started(now)) + }) + .map(|transition| { + transition.members[own] = true; + transition.id + }); + let id = joined.unwrap_or_else(|| { + let id = presentation.next; + presentation.next = Transition(id.0 + 1); + let mut members = [false; 2]; + members[own] = true; + presentation.transitions.push_back(Pending { + id, + members, + resumed: [false; 2], + plans: [None, None], + mapping: None, + }); + id + }); + presentation.declared[own] = id; + drop(presentation); + self.timeline.notify_waiters(); + id + } + + /// Mark where this role's current segment ends on the shared clock. + pub(super) fn end(&self, source: Timestamp) -> anyhow::Result<()> { + let mapping = *self.mapping.lock().unwrap(); + self.presentation.lock().unwrap().ends[self.role as usize] = Some(Boundary { + starts: mapping.starts, + end: timestamp(mapping.map(source)?), + }); + self.timeline.notify_waiters(); + Ok(()) + } + + /// Record the first replacement media for `transition`. + pub(super) fn plan(&self, transition: Transition, source: Timestamp, delay: Option) { + let mut presentation = self.presentation.lock().unwrap(); + let delay = delay.unwrap_or(presentation.delay); + if let Some(pending) = presentation.transition(transition) { + pending.plans[self.role as usize] = Some(Plan { + source: Duration::from(source), + delay, + }); + } + drop(presentation); + self.timeline.notify_waiters(); + } + + /// Media was lost: nothing waits for the discarded tail. + pub(super) fn gap(&self) { + self.presentation.lock().unwrap().ends[self.role as usize] = None; + self.timeline.notify_waiters(); + } + + /// This role will never resume `transition`, so a peer must not pair with its plan. + pub(super) fn abandon(&self, transition: Transition) { + let own = self.role as usize; + let mut presentation = self.presentation.lock().unwrap(); + if let Some(pending) = presentation.transition(transition) { + pending.members[own] = false; + pending.plans[own] = None; + } + presentation.prune(tokio::time::Instant::now().into_std()); + drop(presentation); + self.timeline.notify_waiters(); + } + + pub(super) fn apply(&self, transition: Transition, mapping: Mapping) { + *self.mapping.lock().unwrap() = mapping; + let mut presentation = self.presentation.lock().unwrap(); + if let Some(pending) = presentation.transition(transition) { + pending.resumed[self.role as usize] = true; + } + presentation.prune(tokio::time::Instant::now().into_std()); + drop(presentation); + self.timeline.notify_waiters(); + } + + /// Drain declared old media, then map `transition`'s replacement onto the + /// running clock. + pub(super) async fn resume( + &self, + transition: Transition, + source: Timestamp, + delay: Option, + ) -> anyhow::Result { + loop { + // Armed before reading the boundaries, so a change made between the read + // and the wait still wakes it. + let changed = self.timeline.notified(); + let mut changed = std::pin::pin!(changed); + changed.as_mut().enable(); + let now = tokio::time::Instant::now().into_std(); + let end = { + let mut presentation = self.presentation.lock().unwrap(); + // The peer already resumed this transition after draining every end + // before it; one it declared since must not hold this role back. + if let Some(mapping) = presentation.transition(transition).and_then(|pending| pending.mapping) { + return Ok(mapping); + } + presentation + .ends + .iter() + .flatten() + .filter(|boundary| boundary.starts.is_none_or(|start| start <= now)) + .filter_map(|boundary| presentation.due(media(boundary.end))) + .chain(presentation.tails.iter().copied()) + .max() + }; + if let Some(end) = end.filter(|end| *end > now) { + tokio::select! { + _ = tokio::time::sleep_until(end.into()) => {}, + _ = changed => {}, + } + continue; + } + let mut presentation = self.presentation.lock().unwrap(); + let peer = 1 - self.role as usize; + // A transition abandoned under this resume, by a gap the caller rechecks, + // maps alone like one the peer never joined. + let (mapping, paired) = presentation.transition(transition).map_or((None, None), |pending| { + let both = pending.members[peer] && pending.plans.iter().all(Option::is_some); + (pending.mapping, pending.plans[Kind::Audio as usize].filter(|_| both)) + }); + // The peer already resumed this transition. + if let Some(mapping) = mapping { + return Ok(mapping); + } + // Both roles have replacement media. The speaker's first sample defines + // their common origin even when its adaptive delay differs from video's. + // Otherwise this role maps alone and a peer joining later follows it. + let (source, delay) = match paired { + Some(audio) => (audio.source, audio.delay), + None => (Duration::from(source), delay.unwrap_or(presentation.delay)), + }; + let starts = now + .checked_add(delay) + .context("playback delay exceeds the system clock")?; + let mapping = Mapping { + source, + target: presentation.position(starts).unwrap_or(source), + starts: Some(starts), + }; + if let Some(pending) = presentation.transition(transition) { + pending.mapping = Some(mapping); + } + return Ok(mapping); + } + } +} + +impl Drop for Role { + fn drop(&mut self) { + let own = self.role as usize; + if let Ok(mut presentation) = self.presentation.lock() { + presentation.ends[own] = None; + for pending in presentation + .transitions + .iter_mut() + .filter(|pending| !pending.resumed[own]) + { + pending.members[own] = false; + pending.plans[own] = None; + } + presentation.declared[own] = Transition::default(); + presentation.prune(tokio::time::Instant::now().into_std()); + } + self.timeline.notify_waiters(); + } +} + /// A wire timestamp as a duration from the start of the track. pub(super) fn timestamp(timestamp: Timestamp) -> Duration { Duration::from_micros(timestamp.as_micros().min(u64::MAX as u128) as u64) @@ -238,6 +590,341 @@ mod tests { Timestamp::from_millis(millis).unwrap() } + #[tokio::test] + async fn one_ended_role_maps_to_the_continuous_peers_clock() { + tokio::time::pause(); + let start = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); + presentation.lock().unwrap().video(ms(10_000), start.into_std()); + let transition = audio.declare(); + audio.end(ms(10_020)).unwrap(); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + let unchanged = presentation.lock().unwrap().due(ms(10_500)); + audio.apply(transition, audio.resume(transition, ms(0), Some(DELAY)).await.unwrap()); + assert_eq!(audio.map(ms(0)).unwrap().as_millis(), 10_120); + assert_eq!(video.map(ms(10_500)).unwrap().as_millis(), 10_500); + assert_eq!(presentation.lock().unwrap().due(ms(10_500)), unchanged); + } + + #[tokio::test] + async fn a_late_paired_replacement_keeps_its_media_offset() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); + let transition = audio.declare(); + assert_eq!(video.declare(), transition, "one restart"); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + audio.plan(transition, ms(0), Some(DELAY)); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + audio.apply(transition, audio.resume(transition, ms(0), Some(DELAY)).await.unwrap()); + // The video replacement lands after the resumed audio has started playing. + tokio::time::advance(DELAY * 2).await; + video.plan(transition, ms(40), None); + video.apply(transition, video.resume(transition, ms(40), None).await.unwrap()); + assert_eq!( + video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + 40 + ); + } + + #[tokio::test] + async fn a_late_replacement_pairs_with_its_own_transition() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + video.plan(transition, ms(40), None); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + video.apply(transition, video.resume(transition, ms(40), None).await.unwrap()); + // Video already queued its next replacement when audio's first one lands. + let next = video.declare(); + assert_ne!(next, transition); + video.end(ms(60)).unwrap(); + video.plan(next, ms(5_000), None); + audio.plan(transition, ms(0), Some(DELAY)); + audio.apply(transition, audio.resume(transition, ms(0), Some(DELAY)).await.unwrap()); + assert_eq!( + video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + 40 + ); + } + + #[tokio::test] + async fn a_peer_two_restarts_ahead_pairs_with_the_first() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + video.plan(transition, ms(40), None); + let first = video.resume(transition, ms(40), None).await.unwrap(); + video.apply(transition, first); + // Video restarts again and resumes before audio's first replacement lands. + let next = video.declare(); + video.plan(next, ms(5_000), None); + tokio::time::advance(Duration::from_millis(10)).await; + video.apply(next, video.resume(next, ms(5_000), None).await.unwrap()); + audio.plan(transition, ms(0), Some(DELAY)); + audio.apply(transition, audio.resume(transition, ms(0), Some(DELAY)).await.unwrap()); + assert_eq!( + first.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + 40 + ); + } + + #[tokio::test] + async fn an_abandoned_replacement_leaves_the_next_transition_alone() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + // Video's first replacement is evicted before it plays, then it restarts again. + video.plan(transition, ms(40), None); + video.abandon(transition); + let next = video.declare(); + video.plan(next, ms(5_000), None); + audio.plan(transition, ms(0), Some(DELAY)); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + audio.apply(transition, audio.resume(transition, ms(0), Some(DELAY)).await.unwrap()); + video.apply(next, video.resume(next, ms(5_000), None).await.unwrap()); + // The later replacement maps onto the running clock, not the audio restart's. + let resumed = tokio::time::Instant::now() + DELAY; + assert_eq!( + presentation.lock().unwrap().due(video.map(ms(5_000)).unwrap()), + Some(resumed.into_std()) + ); + } + + #[tokio::test] + async fn an_abandoned_peer_plan_is_not_paired() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + audio.plan(transition, ms(0), Some(Duration::from_millis(80))); + audio.abandon(transition); + video.plan(transition, ms(40), None); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + video.apply(transition, video.resume(transition, ms(40), None).await.unwrap()); + let resumed = tokio::time::Instant::now() + DELAY; + assert_eq!( + presentation.lock().unwrap().due(video.map(ms(40)).unwrap()), + Some(resumed.into_std()) + ); + } + + #[tokio::test] + async fn a_resumed_transition_ignores_the_peers_later_end() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + video.plan(transition, ms(40), None); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + let mapping = video.resume(transition, ms(40), None).await.unwrap(); + video.apply(transition, mapping); + // The resumed video plays, then its encoder stops again far ahead. + tokio::time::advance(DELAY * 2).await; + video.declare(); + video.end(ms(60_000)).unwrap(); + audio.plan(transition, ms(0), Some(DELAY)); + let resumed = + tokio::time::timeout(Duration::from_millis(1), audio.resume(transition, ms(0), Some(DELAY))).await; + assert!(resumed.is_ok(), "audio waited for video's next end"); + } + + #[tokio::test] + async fn a_cancelled_role_releases_its_declared_end() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); + audio.declare(); + audio.end(ms(10_200)).unwrap(); + // Cancelled with its sink, so nothing plays the tail. + drop(audio); + let transition = video.declare(); + video.end(ms(10_000)).unwrap(); + video.plan(transition, ms(0), None); + let resume = tokio::time::timeout( + DELAY + Duration::from_millis(150), + video.resume(transition, ms(0), None), + ) + .await; + assert!(resume.is_ok(), "video waited for discarded audio"); + } + + #[tokio::test] + async fn a_closed_role_still_holds_its_declared_end() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); + audio.declare(); + audio.end(ms(10_200)).unwrap(); + // The track closed, but its returned sink still plays the tail up to the end. + audio.retire(Some((now + Duration::from_millis(300)).into_std())); + let transition = video.declare(); + video.end(ms(10_000)).unwrap(); + video.plan(transition, ms(0), None); + let resume = tokio::time::timeout( + DELAY + Duration::from_millis(150), + video.resume(transition, ms(0), None), + ) + .await; + assert!(resume.is_err(), "video resumed before the audio tail played"); + } + + #[tokio::test] + async fn a_retired_end_ignores_a_later_re_pin() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); + audio.declare(); + audio.end(ms(10_200)).unwrap(); + audio.retire(Some((now + Duration::from_millis(300)).into_std())); + // A replacement rendition starts over at zero and re-pins the anchor. + { + let mut presentation = presentation.lock().unwrap(); + presentation.restarted(); + presentation.audio(Duration::from_millis(50), DELAY, now.into_std()); + } + let transition = video.declare(); + video.plan(transition, ms(0), None); + let resume = tokio::time::timeout(Duration::from_millis(400), video.resume(transition, ms(0), None)).await; + assert!(resume.is_ok(), "video waited past the drained tail"); + } + + #[tokio::test] + async fn a_cancelled_successor_keeps_a_retired_tail() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); + audio.declare(); + audio.end(ms(10_200)).unwrap(); + audio.retire(Some((now + Duration::from_millis(300)).into_std())); + // The next rendition fails while the retired sink still plays. + let successor = Role::new(presentation.clone(), Kind::Audio); + successor.declare(); + drop(successor); + let transition = video.declare(); + video.plan(transition, ms(0), None); + let resume = tokio::time::timeout(Duration::from_millis(250), video.resume(transition, ms(0), None)).await; + assert!(resume.is_err(), "video resumed over the retired audio tail"); + } + + #[tokio::test] + async fn paired_plans_keep_media_offsets_and_follow_the_speakers_delay() { + tokio::time::pause(); + let now = tokio::time::Instant::now(); + let presentation = std::sync::Arc::new(std::sync::Mutex::new(Presentation::new(DELAY))); + presentation + .lock() + .unwrap() + .audio(Duration::from_secs(10), DELAY, now.into_std()); + let audio = Role::new(presentation.clone(), Kind::Audio); + let video = Role::new(presentation.clone(), Kind::Video); + let transition = audio.declare(); + video.declare(); + audio.end(ms(10_020)).unwrap(); + video.end(ms(10_020)).unwrap(); + audio.plan(transition, ms(0), Some(Duration::from_millis(80))); + video.plan(transition, ms(40), None); + tokio::time::advance(DELAY + Duration::from_millis(20)).await; + video.apply(transition, video.resume(transition, ms(40), None).await.unwrap()); + audio.apply( + transition, + audio + .resume(transition, ms(0), Some(Duration::from_millis(80))) + .await + .unwrap(), + ); + assert_eq!( + video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + 40 + ); + assert_eq!( + presentation.lock().unwrap().due(audio.map(ms(0)).unwrap()), + Some((now + Duration::from_millis(200)).into_std()) + ); + } + /// Earlier arrivals pull the anchor forward so playback converges on the live /// edge without giving up the configured delay. #[test] diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 7827dad4a61..9d7d1794026 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -9,7 +9,7 @@ use tokio::time::Instant; use super::buffer::Buffer; use super::output::Output; -use super::timeline::Presentation; +use super::timeline::{Presentation, Role, Transition}; use super::window::Event; /// A few surfaces, independent of the requested playout delay. A codec batch @@ -26,6 +26,7 @@ enum Step { Decode, /// The track ended: drain what the codec still holds. Flush, + Resume, } /// When the window's queue has room for another picture, or `None` if it does @@ -69,50 +70,145 @@ impl Decoder for moq_video::decode::Sink { } } +/// One encoder segment's media, and the transition it resumes when it +/// replaces the segment before it. +#[derive(Default)] +struct Segment { + media: Buffer, + resumes: Option, + /// When its first picture arrived. + arrived: Option, +} + +impl std::ops::Deref for Segment { + type Target = Buffer; + + fn deref(&self) -> &Buffer { + &self.media + } +} + +impl std::ops::DerefMut for Segment { + fn deref_mut(&mut self) -> &mut Buffer { + &mut self.media + } +} + impl Video { pub async fn run(self, mut track: Track, mut decoder: impl Decoder) -> anyhow::Result<()> { - let buffer = Mutex::new(Buffer::default()); + let closed = std::sync::atomic::AtomicBool::new(false); + let advancing = std::sync::atomic::AtomicBool::new(false); + let revision = std::sync::atomic::AtomicU64::new(0); + let buffers = Mutex::new(VecDeque::from([Segment::default()])); + // Signalled when a segment held behind a resume becomes the front. + let advanced = tokio::sync::Notify::new(); + let role = Role::new(self.presentation.clone(), super::playback::Kind::Video); let receive = async { // The group of a declared end whose restart waits for a later group, so the - // media buffered before the end still plays out. + // media buffered before the end still plays out, and the transition that + // later group resumes. let mut end = None; + loop { + // Stop reading while media queued behind a resume arrived more than the age + // budget ago, so the subscription evicts the excess instead of the queue + // growing a segment per restart without bound. + while buffers + .lock() + .unwrap() + .iter() + .skip(1) + .filter(|segment| segment.oldest().is_some()) + .filter_map(|segment| segment.arrived) + .min() + .is_some_and(|arrived| arrived.elapsed() > self.max_delay) + { + advanced.notified().await; + } let read = track.read().await?; - let mut buffer = buffer.lock().unwrap(); - let before = buffer.generation; + let mut buffers = buffers.lock().unwrap(); + let active = buffers.len() == 1 && !advancing.load(std::sync::atomic::Ordering::Relaxed); + let before = buffers.front().unwrap().generation; let ended = match read { Some(Read::Frame { group, frame }) => { - if end.take_if(|end| *end != group).is_some() { - buffer.clear(); - self.presentation.lock().unwrap().video_restarted(); + if let Some((_, transition)) = end.take_if(|(end, _)| *end != group) { + // A superseded segment never receives another picture, so one that + // eviction emptied can never resume. + let mut index = 1; + while index < buffers.len() { + if buffers[index].oldest().is_some() { + index += 1; + } else if let Some(skipped) = buffers.remove(index).and_then(|skipped| skipped.resumes) + { + role.abandon(skipped); + } + } + role.plan(transition, frame.timestamp, None); + buffers.push_back(Segment { + resumes: Some(transition), + ..Default::default() + }); } - if self + if active + && buffers.len() == 1 && self .presentation .lock() .unwrap() - .video(frame.timestamp, Instant::now().into_std()) + .video(role.map(frame.timestamp)?, Instant::now().into_std()) { self.output.send(Event::Wake); } - buffer.push(frame, self.max_delay); + let back = buffers.back_mut().unwrap(); + back.push(frame, self.max_delay); + back.arrived.get_or_insert_with(Instant::now); false } // Applied in delivery order, so the restart lands before the frame after it. + // An end declared before the loss still holds, as on the audio side: what + // follows comes from the replacement encoder and resumes the latest + // transition, while the media lost with the gap is discarded. Some(Read::Gap { .. }) => { - end = None; - buffer.clear(); + let mut queued: Vec<_> = buffers.iter().filter_map(|buffer| buffer.resumes).collect(); + let resumes = queued.pop(); + for transition in queued { + role.abandon(transition); + } + revision.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + role.gap(); + buffers.clear(); + buffers.push_back(Segment::default()); + let front = buffers.front_mut().unwrap(); + front.clear(); + front.generation = before + 1; + front.ended = resumes.is_some() || end.is_some(); + if resumes.is_some() { + buffers.push_back(Segment { + resumes, + ..Default::default() + }); + } self.presentation.lock().unwrap().video_restarted(); false } - Some(Read::End { group, .. }) => { - end = Some(group); + Some(Read::End { group, end: endpoint }) => { + if let Some((_, transition)) = end.take() { + role.abandon(transition); + } + end = Some((group, role.declare())); + buffers.back_mut().unwrap().ended = true; + buffers.back_mut().unwrap().end = Some(endpoint); + if active { + role.end(endpoint)?; + } false } None => { - buffer.ended = true; + closed.store(true, std::sync::atomic::Ordering::Relaxed); + buffers.back_mut().unwrap().ended = true; true } }; + let buffer = buffers.front().unwrap(); if buffer.generation != before { self.frames.lock().unwrap().clear(); self.output.send(Event::Wake); @@ -136,9 +232,10 @@ impl Video { let mut held = BTreeSet::new(); // Decoded pictures waiting for room in the window's queue. let mut pending = VecDeque::new(); - loop { + let drained = loop { let (current, oldest, ended) = { - let buffer = buffer.lock().unwrap(); + let buffers = buffers.lock().unwrap(); + let buffer = buffers.front().unwrap(); (buffer.generation, buffer.oldest(), buffer.ended) }; if generation.is_some_and(|generation| generation != current) { @@ -171,6 +268,7 @@ impl Video { // Every access unit up to the earliest picture still owed must be // decoded before that picture is due, however deep the stream // reorders or the codec holds pictures back. + let oldest = role.map(oldest)?; let owed = held.first().map_or(oldest, |held| oldest.min(*held)); let at = presentation .due(owed) @@ -185,7 +283,20 @@ impl Video { } else if generation.is_some() { Step::Flush } else { - break; + // A new encoder waits for every old picture to be presented. A closed + // track hands its last due time to whatever resumes after it. + if closed.load(std::sync::atomic::Ordering::Relaxed) && buffers.lock().unwrap().len() == 1 { + break queue.back().and_then(|frame| presentation.due(frame.timestamp)); + } else if !queue.is_empty() { + Step::Wait( + queue + .back() + .and_then(|frame| presentation.due(frame.timestamp)) + .filter(|at| *at > now), + ) + } else { + Step::Resume + } } }; let frames = match step { @@ -201,21 +312,76 @@ impl Video { continue; } Step::Decode => { - let frame = buffer + let mut frame = buffers .lock() .unwrap() + .front_mut() + .unwrap() .pop() .expect("no await since inspecting the buffer"); + frame.timestamp = role.map(frame.timestamp)?; generation = Some(current); held.insert(frame.timestamp); // Never race a codec call: cancelling Sink::decode poisons it. The // joined receiver continues observing arrivals during this await. decoder.decode(frame).await? } + Step::Resume => { + // Read under the same lock as the pop: a gap replaces the buffers and + // bumps the revision while holding it. + let next = { + let mut buffers = buffers.lock().unwrap(); + // Eviction can empty a replacement until its next keyframe; one a + // later replacement already follows never gets another. + while buffers.len() > 2 && buffers[1].oldest().is_none() { + if let Some(transition) = buffers.remove(1).and_then(|skipped| skipped.resumes) { + role.abandon(transition); + } + } + match buffers.get(1).and_then(|next| Some((next.oldest()?, next.resumes?))) { + Some((start, transition)) => { + advancing.store(true, std::sync::atomic::Ordering::Relaxed); + buffers.pop_front(); + advanced.notify_one(); + Some((start, transition, revision.load(std::sync::atomic::Ordering::Relaxed))) + } + None => None, + } + }; + let Some((start, transition, before)) = next else { + if closed.load(std::sync::atomic::Ordering::Relaxed) { + break None; + } + self.changed.notified().await; + continue; + }; + let mapping = role.resume(transition, start, None).await?; + let buffers = buffers.lock().unwrap(); + if revision.load(std::sync::atomic::Ordering::Relaxed) != before { + advancing.store(false, std::sync::atomic::Ordering::Relaxed); + continue; + } + role.apply(transition, mapping); + if let Some(end) = buffers.front().unwrap().end { + role.end(end)?; + } + advancing.store(false, std::sync::atomic::Ordering::Relaxed); + let mut presentation = self.presentation.lock().unwrap(); + for frame in &buffers.front().unwrap().frames { + presentation.video(role.map(frame.timestamp)?, Instant::now().into_std()); + } + self.output.send(Event::Wake); + continue; + } Step::Flush => { - generation = None; + let flushed = generation.take(); held.clear(); - decoder.flush().await? + let frames = decoder.flush().await?; + // A gap during the await already discarded these pictures' generation. + if flushed != Some(buffers.lock().unwrap().front().unwrap().generation) { + continue; + } + frames } }; // A codec returns display order, so a picture held before the newest @@ -225,16 +391,24 @@ impl Video { } // A codec may return pictures it held across the keyframe that // resumed a skipped timeline. Those pictures no longer have a slot. - let floor = buffer.lock().unwrap().floor; + let floor = buffers + .lock() + .unwrap() + .front() + .unwrap() + .floor + .map(|floor| role.map(floor)) + .transpose()?; pending.extend( frames .into_iter() .filter(|frame| floor.is_none_or(|floor| frame.timestamp.as_micros() >= floor.as_micros())), ); - } - Ok::<_, anyhow::Error>(()) + }; + Ok::<_, anyhow::Error>(drained) }; - tokio::try_join!(receive, decode)?; + let (_, drained) = tokio::try_join!(receive, decode)?; + role.retire(drained); Ok(()) } } @@ -443,7 +617,7 @@ mod tests { assert_eq!(shown, [0, 33, 66, 99], "reordered output was not presented in order"); } #[tokio::test] - async fn a_discontinuity_discards_old_pictures_and_restarts_the_delay() { + async fn a_gap_discards_old_pictures_and_restarts_the_delay() { tokio::time::pause(); let delay = Duration::from_secs(2); let media = media(delay); @@ -472,7 +646,7 @@ mod tests { tokio::time::advance(delay).await; tokio::task::yield_now().await; assert_eq!(decoded.lock().unwrap().len(), 1, "old picture held inside decoder"); - producer.discontinuity().unwrap(); + producer.seek(2).unwrap(); producer.write(frame(500, true)).unwrap(); producer.finish().unwrap(); let restarted = Instant::now(); @@ -597,6 +771,284 @@ mod tests { task.await.unwrap().unwrap(); } + #[tokio::test] + async fn a_closed_track_holds_its_end_until_its_last_picture_is_due() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let media = media(delay); + let broadcast = moq_net::broadcast::Info::new().produce(); + let source = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = source + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(delay)) + .await + .unwrap(); + let format = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let mut producer = moq_mux::container::Producer::new(source, format()); + for ms in [0, 33, 66] { + producer.write(frame(ms, ms == 0)).unwrap(); + } + // The publisher declares an end, then the track closes. + producer.discontinuity().unwrap(); + producer.finish().unwrap(); + let track = moq_mux::container::Consumer::new(subscriber, format()); + // The window never presents, so the decoded pictures stay queued. + let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0))); + while !task.is_finished() { + tokio::time::advance(Duration::from_millis(1)).await; + tokio::task::yield_now().await; + } + task.await.unwrap().unwrap(); + assert_eq!(media.video.lock().unwrap().len(), 3, "the pictures were presented"); + let last = media + .presentation + .lock() + .unwrap() + .due(moq_net::Timestamp::from_millis(66).unwrap()) + .unwrap(); + assert!( + Instant::now().into_std() < last, + "the track closed after its tail was due" + ); + let audio = Role::new(media.presentation.clone(), super::super::playback::Kind::Audio); + let transition = audio.declare(); + audio.resume(transition, moq_net::Timestamp::ZERO, None).await.unwrap(); + assert!(Instant::now().into_std() >= last, "audio resumed over the video tail"); + } + + #[tokio::test] + async fn a_held_video_resume_leaves_the_excess_to_the_subscription() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let media = media(delay); + let broadcast = moq_net::broadcast::Info::new().produce(); + let source = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = source + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(delay)) + .await + .unwrap(); + let format = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let mut producer = moq_mux::container::Producer::new(source, format()); + let track = moq_mux::container::Consumer::new(subscriber, format()); + let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0))); + let audio = Role::new(media.presentation.clone(), super::super::playback::Kind::Audio); + // Two-picture segments, one per restart, published live for two seconds + // while audio's end holds the first replacement back. + const SEGMENTS: u64 = 50; + let mut shown = 0; + // Presents until the track drains, which replaying every queued segment would + // take far longer than this bound. + for at in 0..60_000u64 { + if task.is_finished() { + break; + } + if at < 60 && at % 20 == 0 { + producer.write(frame(at, at == 0)).unwrap(); + } else if at == 60 { + audio.declare(); + audio.end(moq_net::Timestamp::from_millis(2_000).unwrap()).unwrap(); + } + if (60..60 + SEGMENTS * 40).contains(&at) && at % 20 == 0 { + if at % 40 == 20 { + producer.discontinuity().unwrap(); + } + producer.write(frame(at, at % 40 == 20)).unwrap(); + } else if at == 60 + SEGMENTS * 40 { + producer.finish().unwrap(); + } + tokio::task::yield_now().await; + if media + .output + .present(&media) + .is_some_and(|picture| picture.as_millis() >= 60) + { + shown += 1; + } + tokio::time::advance(Duration::from_millis(1)).await; + } + task.await.unwrap().unwrap(); + assert!(shown > 0, "the replacements played"); + assert!( + shown < SEGMENTS, + "pictures held behind the resume past the age budget were queued, not evicted: {shown}" + ); + } + + #[tokio::test] + async fn an_emptied_superseded_segment_is_discarded() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let media = media(delay); + let broadcast = moq_net::broadcast::Info::new().produce(); + let source = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + // The network keeps every group; only the player's age budget evicts. + let subscriber = source + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))) + .await + .unwrap(); + let format = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let mut producer = moq_mux::container::Producer::new(source, format()); + let track = moq_mux::container::Consumer::new(subscriber, format()); + let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0))); + // Audio's end two seconds out holds every replacement back. + let audio = Role::new(media.presentation.clone(), super::super::playback::Kind::Audio); + audio.declare(); + audio.end(moq_net::Timestamp::from_millis(2_000).unwrap()).unwrap(); + producer.write(frame(0, true)).unwrap(); + // Each replacement is one GOP longer than the age budget, so eviction + // takes its only keyframe and leaves it empty. + const SEGMENTS: u64 = 10; + for segment in 0..SEGMENTS { + producer.discontinuity().unwrap(); + for index in 0..15 { + producer + .write(frame(20 + segment * 300 + index * 20, index == 0)) + .unwrap(); + } + } + tokio::time::advance(Duration::from_millis(500)).await; + tokio::task::yield_now().await; + let pending = media.presentation.lock().unwrap().pending(); + assert!(pending <= 3, "emptied segments kept their transitions: {pending}"); + producer.finish().unwrap(); + drop(audio); + for _ in 0..1_000 { + tokio::task::yield_now().await; + media.output.present(&media); + tokio::time::advance(Duration::from_millis(1)).await; + } + task.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn a_waiting_resume_leaves_the_decoders_wakeup_alone() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let media = media(delay); + let broadcast = moq_net::broadcast::Info::new().produce(); + let source = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = source + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(delay)) + .await + .unwrap(); + let format = || moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let mut producer = moq_mux::container::Producer::new(source, format()); + let track = moq_mux::container::Consumer::new(subscriber, format()); + let decoder = Buffered::new(0); + let decoded = decoder.decoded.clone(); + let task = tokio::spawn(playback(&media, delay).run(track, decoder)); + for ms in [0, 20, 40] { + producer.write(frame(ms, ms == 0)).unwrap(); + } + // Present until `count` pictures are decoded, or give up after 20 s. + let step = async |count: usize| { + for _ in 0..20_000 { + if decoded.lock().unwrap().len() >= count { + break; + } + tokio::task::yield_now().await; + media.output.present(&media); + tokio::time::advance(Duration::from_millis(1)).await; + } + }; + step(3).await; + // Audio waits on an end ten seconds out while the decoder idles. + let audio = Role::new(media.presentation.clone(), super::super::playback::Kind::Audio); + let transition = audio.declare(); + audio.end(moq_net::Timestamp::from_secs(10).unwrap()).unwrap(); + let resume = tokio::spawn(async move { + audio + .resume(transition, moq_net::Timestamp::ZERO, None) + .await + .map(|_| ()) + }); + for _ in 0..200 { + tokio::task::yield_now().await; + media.output.present(&media); + tokio::time::advance(Duration::from_millis(1)).await; + } + // The publisher sends one more picture after the pause. + producer.write(frame(60, false)).unwrap(); + let sent = Instant::now(); + step(4).await; + assert!( + sent.elapsed() < Duration::from_millis(50), + "the decoder missed the new picture for {:?}", + sent.elapsed() + ); + producer.finish().unwrap(); + resume.abort(); + for _ in 0..500 { + tokio::task::yield_now().await; + media.output.present(&media); + tokio::time::advance(Duration::from_millis(1)).await; + } + task.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn a_quick_resume_flushes_and_plays_every_old_picture_first() { + tokio::time::pause(); + let delay = Duration::from_millis(200); + let media = media(delay); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = track + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(delay)) + .await + .unwrap(); + let format = moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + ); + for ms in [0, 33, 66] { + producer.write(frame(ms, ms == 0)).unwrap(); + } + producer.discontinuity().unwrap(); + for ms in [0, 33, 66] { + producer.write(frame(ms, ms == 0)).unwrap(); + } + producer.finish().unwrap(); + let decoder = Buffered::new(2); + let flushed = decoder.flushed.clone(); + let task = + tokio::spawn(playback(&media, delay).run(moq_mux::container::Consumer::new(subscriber, format), decoder)); + let mut shown = Vec::new(); + for _ in 0..1_000 { + tokio::task::yield_now().await; + if let Some(timestamp) = media.output.present(&media) { + shown.push(timestamp.as_millis()); + } + tokio::time::advance(Duration::from_millis(1)).await; + } + task.await.unwrap().unwrap(); + assert_eq!(&shown[..3], [0, 33, 66]); + assert_eq!(shown.len(), 6); + assert!(shown[3] > shown[2], "the resumed segment follows every old picture"); + assert_eq!(shown[4] - shown[3], 33); + assert_eq!(shown[5] - shown[4], 33); + assert_eq!( + *flushed.lock().unwrap(), + 2, + "each encoder drains before its replacement" + ); + } + /// Play one burst of 33 ms pictures, given by slot in decode order, through a /// codec holding `depth` pictures, and check the window shows every one on /// time. diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index 911835f6269..4c5a61171aa 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -407,7 +407,13 @@ struct ConsumerInner { impl ConsumerInner { async fn next(&mut self) -> Result, MoqError> { - Ok(self.consumer.read().await?.map(Into::into)) + loop { + match self.consumer.read().await? { + Some(moq_audio::decode::Read::Frame { frame, .. }) => return Ok(Some(frame.into())), + Some(moq_audio::decode::Read::End { .. }) => {} + None => return Ok(None), + } + } } }