From 49c7a1e19761cee9ef3d6cfe5911c223104bdf06 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 14:04:26 -0700 Subject: [PATCH 01/10] chore(quest): claim Rust frame continuity signaling Co-Authored-By: GPT-6 From f0dc21e28c3c683f8774c35d69d240f26d65faf7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 15:14:44 -0700 Subject: [PATCH 02/10] feat(mux): expose container and audio delivery continuity Co-Authored-By: GPT-6 --- doc/lib/rs/moq-mux.md | 4 + quest/m1/rust-continuous.md | 15 +++ rs/hang/examples/subscribe.rs | 6 +- rs/moq-audio/src/decode/consumer.rs | 135 ++++++++++++++----- rs/moq-audio/src/encode/capture.rs | 2 +- rs/moq-audio/src/encode/producer.rs | 29 ++-- rs/moq-audio/src/playback.rs | 2 +- rs/moq-audio/tests/import.rs | 2 +- rs/moq-audio/tests/roundtrip.rs | 7 +- rs/moq-c/src/audio.rs | 2 +- rs/moq-c/src/consume.rs | 2 +- rs/moq-cli/src/play/media.rs | 2 +- rs/moq-cli/src/play/video.rs | 3 +- rs/moq-cli/src/publish.rs | 3 +- rs/moq-ffi/src/audio.rs | 2 +- rs/moq-ffi/src/media.rs | 6 +- rs/moq-gst/src/source/imp.rs | 2 +- rs/moq-mux/src/codec/opus/import.rs | 4 +- rs/moq-mux/src/codec/vp8/import.rs | 4 +- rs/moq-mux/src/container/consumer.rs | 140 +++++++++++++------- rs/moq-mux/src/container/flv/import_test.rs | 20 ++- rs/moq-mux/src/container/group.rs | 6 +- rs/moq-mux/src/container/mkv/import_test.rs | 2 +- rs/moq-mux/src/container/mod.rs | 9 ++ rs/moq-mux/src/container/producer.rs | 2 +- rs/moq-mux/src/container/test_util.rs | 2 +- rs/moq-mux/src/container/ts/export_test.rs | 8 +- rs/moq-mux/src/container/ts/import.rs | 24 ++-- rs/moq-mux/src/container/ts/import_test.rs | 13 +- rs/moq-mux/src/container/ts/programs.rs | 7 +- rs/moq-mux/src/import/track.rs | 10 +- rs/moq-rtc/src/codec/mod.rs | 8 +- rs/moq-srt/src/ts.rs | 9 +- rs/moq-video/src/capture/wgc.rs | 2 +- rs/moq-video/src/decode/consumer.rs | 6 +- rs/moq-video/src/decode/decoder.rs | 4 +- rs/moq-video/src/encode/producer.rs | 2 +- 37 files changed, 341 insertions(+), 165 deletions(-) diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index 814518cff1e..e90fba94072 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -30,6 +30,10 @@ set at the init segment, so a new rendition or a changed configuration ends the export. The [CLI page](/bin/cli#import) covers these and the MPEG-TS specifics: one program per broadcast, damaged packets, and feed checks. +Container and audio reads return a frame with its delivery continuity: the first +frame and the first after a skip, hole, or declared break start a new span. +Video decoders return bare frames. + Data tracks are catalog entries too. The [hang page](/concept/hang#data-tracks) describes the modes. A capture time converts onto the broadcast clock; a timestamp taken from the source is published as given and lines up with media diff --git a/quest/m1/rust-continuous.md b/quest/m1/rust-continuous.md index a6032b7b82d..c1448c53401 100644 --- a/quest/m1/rust-continuous.md +++ b/quest/m1/rust-continuous.md @@ -28,6 +28,21 @@ continuous. Public API: breaking in moq-mux (the consumer's read result changes). Wire: none. +### Draft progress and remaining video propagation + +The draft adds a shared Sample envelope to container and audio reads, with +regressions for subscribe, markers, holes, skips, and audio decode. Existing +consumers explicitly unwrap the frame; binding surfaces stay unchanged. + +Video still returns its existing decoded Frame. A decoder can return a picture +from before a marker while processing the access unit after it, or return it +only on flush. A single pending boolean attaches the new boundary to the old +picture and loses the new picture boundary, so that approach is not included. +Carry input continuity through codec output association before extending the +video read result. Prefer private decoder metadata association over a +timestamp heuristic, and cover delayed output on both sides of a marker. +The video propagation goal remains open here. + ## Related - [Consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md) - withholds frames after the same signal diff --git a/rs/hang/examples/subscribe.rs b/rs/hang/examples/subscribe.rs index b929f4b8c7e..9622449b2e0 100644 --- a/rs/hang/examples/subscribe.rs +++ b/rs/hang/examples/subscribe.rs @@ -117,9 +117,9 @@ async fn play(consumer: moq_net::origin::Consumer, announce: moq_net::announce:: // Read frames in latency-bounded presentation order. while let Some(frame) = ordered.read().await? { tracing::info!( - timestamp = ?frame.timestamp, - keyframe = frame.keyframe, - bytes = frame.payload.len(), + timestamp = ?frame.frame.timestamp, + keyframe = frame.frame.keyframe, + bytes = frame.frame.payload.len(), "received frame" ); } diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 8875b579724..70f444e4755 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -1,5 +1,6 @@ //! Subscribe to an encoded audio track and emit raw PCM. +use moq_mux::container::Sample; use std::collections::VecDeque; use std::time::{Duration, Instant}; @@ -84,7 +85,7 @@ pub struct Consumer { next_start: Option, /// Frames decoded and not yet handed back, so a gap's tail can be returned /// ahead of the packet that exposed it. - ready: VecDeque, + ready: VecDeque>, /// Codec activity spans the resampler's buffered output still covers. spans: VecDeque, /// Activity of the last span the output ran past, for the rounding samples the @@ -103,6 +104,7 @@ pub struct Consumer { terminal_start: Option, /// Last container playhead generation applied to timeline state. discontinuity: u64, + continuous: bool, /// The playout delay, fixed or estimated from arrivals. target: Target, /// Where arrival times are measured from. @@ -208,6 +210,7 @@ impl Consumer { end: None, terminal_start: None, discontinuity: 0, + continuous: false, origin: now(), }) } @@ -250,6 +253,7 @@ impl Consumer { } /// Read the next decoded PCM frame, or `None` when the track ends. + /// The returned sample carries the container's delivery continuity through codec buffering. /// /// [`Frame::activity`] reports whether the packet these samples came from /// coded audio. It describes where the frame begins, so a resampled @@ -261,7 +265,7 @@ impl Consumer { /// side stay anchored to their own packet timeline, so the hole is there to /// see. "Doesn't continue" allows for the quantization the stamps carry, which /// on a millisecond-stamped ingest is most of a millisecond. - pub async fn read(&mut self) -> Result, Error> { + pub async fn read(&mut self) -> Result>, Error> { loop { if let Some(frame) = self.ready.pop_front() { return Ok(Some(frame)); @@ -269,10 +273,14 @@ impl Consumer { let mux_frame = self.track.read().await?; self.apply_discontinuity()?; - let Some(mux_frame) = mux_frame else { - return self.flush(); + let Some(sample) = mux_frame else { + return Ok(self.flush()?.map(|frame| { + let continuous = std::mem::replace(&mut self.continuous, true); + Sample { frame, continuous } + })); }; + let mux_frame = sample.frame; // Every packet the container hands over is an arrival, including one the // decoder is about to refuse, so it is observed before anything can fail. let arrival = jitter::millis(now().duration_since(self.origin)); @@ -301,9 +309,11 @@ impl Consumer { .is_some_and(|next| discontinuous(next, mux_frame.timestamp)) && let Some(frame) = self.gap()? { - self.ready.push_back(frame); + let continuous = std::mem::replace(&mut self.continuous, true); + self.ready.push_back(Sample { frame, continuous }); } + self.continuous &= sample.continuous; let rate = self.decoder.sample_rate(); let epoch = *self.epoch.get_or_insert(mux_frame.timestamp); let delay = self.decoder.delay_remaining(); @@ -397,7 +407,8 @@ impl Consumer { // Queued rather than returned, so a tail drained at a gap earlier in this // same iteration still comes out first. The next turn of the loop pops it. let frame = self.frame(pcm, timestamp, activity)?; - self.ready.push_back(frame); + let continuous = std::mem::replace(&mut self.continuous, true); + self.ready.push_back(Sample { frame, continuous }); } } @@ -595,6 +606,51 @@ mod tests { use crate::encode::{Encoder, Input, Options as EncodeOptions, Producer, Settings}; use crate::{Format, Layout}; + #[tokio::test] + async fn continuity_survives_audio_decode_and_a_marker() { + use crate::decode::decoder::tests::{opus_catalog, opus_packets}; + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let subscriber = broadcast.consume(); + let catalog = opus_catalog(moq_mux::codec::opus::Config::new(48_000, 1).with_pre_skip(312)); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + let mut consumer = Consumer::new( + &subscriber, + &catalog, + "audio", + Options { + max_delay: std::time::Duration::from_secs(10), + ..Options::new() + }, + ) + .await + .unwrap(); + for (index, payload) in opus_packets(4).into_iter().enumerate() { + if index == 2 { + producer.discontinuity().unwrap(); + } + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_micros(index as u64 * 20_000).unwrap(), + duration: None, + payload, + keyframe: index % 2 == 0, + }) + .unwrap(); + } + producer.finish().unwrap(); + let mut flags = Vec::new(); + while let Some(sample) = consumer.read().await.unwrap() { + flags.push(sample.continuous); + } + assert_eq!(flags, [false, true, false, true]); + } + #[tokio::test] async fn remixes_mono_stream_to_stereo_output() { let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -637,7 +693,7 @@ mod tests { producer.write(&Frame::new(data.into(), Timestamp::ZERO)).unwrap(); let frame = consumer.read().await.unwrap().expect("decoded frame"); - let samples = Format::F32.as_interleaved_f32(&frame.data, 2).unwrap(); + let samples = Format::F32.as_interleaved_f32(&frame.frame.data, 2).unwrap(); assert_eq!(samples.len(), (960 - 312) * 2); for pair in samples.as_chunks::<2>().0.iter() { assert_eq!(pair[0], pair[1]); @@ -681,8 +737,8 @@ mod tests { // 312 samples at 48 kHz is 6.5 ms. for (micros, frames) in [(0, 960 - 312), (13_500, 960), (33_500, 960)] { let frame = consumer.read().await.unwrap().expect("decoded frame"); - assert_eq!(frame.timestamp.as_micros(), micros); - assert_eq!(frame.data.len() / size_of::(), frames); + assert_eq!(frame.frame.timestamp.as_micros(), micros); + assert_eq!(frame.frame.data.len() / size_of::(), frames); } } @@ -737,7 +793,7 @@ mod tests { } let first = consumer.read().await.unwrap().expect("decoded frame"); - assert_eq!(first.timestamp.as_micros(), 0); + assert_eq!(first.frame.timestamp.as_micros(), 0); // Continuity, not a fixed number: the second frame starts where the first // one's samples end, whatever they came to. Within a few frames rather than @@ -745,9 +801,9 @@ mod tests { // wobbles around the nominal ratio; a real hole (the samples it held back, or // the startup silence it dropped) is twenty times this tolerance. let second = consumer.read().await.unwrap().expect("decoded frame"); - let first_frames = (first.data.len() / size_of::()) as u128; + let first_frames = (first.frame.data.len() / size_of::()) as u128; let ends_at = first_frames * 1_000_000 / 48_000; - let gap = second.timestamp.as_micros().abs_diff(ends_at); + let gap = second.frame.timestamp.as_micros().abs_diff(ends_at); assert!(gap < 100, "expected the frames to meet, got a {gap} us gap"); } @@ -797,10 +853,10 @@ mod tests { producer.finish().unwrap(); let first = consumer.read().await.unwrap().expect("decoded frame"); - let first_frames = first.data.len() / size_of::(); + let first_frames = first.frame.data.len() / size_of::(); let tail = consumer.read().await.unwrap().expect("flushed tail"); - let tail_frames = tail.data.len() / size_of::(); + let tail_frames = tail.frame.data.len() / size_of::(); // The 142 held-back frames at 44.1 kHz are ~155 at 48 kHz, plus the 69 the // sinc filter still owes: it runs centred, so the end of the track only @@ -809,7 +865,7 @@ mod tests { // It picks up where the first frame's samples ended, within the same few // frames of whole-frame rounding as above. let ends_at = (first_frames as u128) * 1_000_000 / 48_000; - let gap = tail.timestamp.as_micros().abs_diff(ends_at); + let gap = tail.frame.timestamp.as_micros().abs_diff(ends_at); assert!(gap < 100, "expected the tail to meet the body, got a {gap} us gap"); // Together they cover the packet and no more: 1024 frames at 44.1 kHz is @@ -884,9 +940,9 @@ mod tests { // nothing every other packet. Those must not surface as frames: a frame // with no samples reads as audio arriving, and carries an activity // describing samples that are not there. - assert!(!frame.data.is_empty(), "read returned a frame with no samples"); - if frame.activity.is_dtx() { - actual = Some(frame.timestamp); + assert!(!frame.frame.data.is_empty(), "read returned a frame with no samples"); + if frame.frame.activity.is_dtx() { + actual = Some(frame.frame.timestamp); break; } } @@ -954,7 +1010,10 @@ mod tests { let mut read = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - read.push((frame.timestamp.as_micros(), frame.data.len() / size_of::())); + read.push(( + frame.frame.timestamp.as_micros(), + frame.frame.data.len() / size_of::(), + )); } read } @@ -1105,9 +1164,9 @@ mod tests { // The output begins with the first packet's samples, so it is stamped and // labelled from that packet. Rewinding from the terminal one instead drops it // most of a second into the future, carrying the DTX label with it. - assert_eq!(frame.timestamp.as_micros(), 0, "held samples moved with the jump"); + assert_eq!(frame.frame.timestamp.as_micros(), 0, "held samples moved with the jump"); assert!( - frame.activity.is_active(), + frame.frame.activity.is_active(), "held samples took the terminal packet's label" ); } @@ -1205,20 +1264,24 @@ mod tests { // shortfall is codec delay, not a hole: without counting it the packet after // every stream start would read as a gap. let first = consumer.read().await.unwrap().expect("decoded frame"); - let frames = first.data.len() / size_of::(); + let frames = first.frame.data.len() / size_of::(); assert!(frames < 960, "the pre-skip should be trimmed, got {frames} frames"); // The hole is real, so codec prediction starts over but stream-level pre-skip // does not. The audio is stamped where the packet says rather than 2.5 ms early. let second = consumer.read().await.unwrap().expect("decoded frame"); - assert_eq!(second.timestamp.as_micros(), 22_500); - assert_eq!(second.data.len() / size_of::(), 960, "pre-skip was reapplied"); + assert_eq!(second.frame.timestamp.as_micros(), 22_500); + assert_eq!( + second.frame.data.len() / size_of::(), + 960, + "pre-skip was reapplied" + ); let third = consumer.read().await.unwrap().expect("decoded frame after gap"); - let second_frames = second.data.len() / size_of::(); + let second_frames = second.frame.data.len() / size_of::(); assert_eq!( - third.timestamp, - advance(second.timestamp, second_frames, 48_000).unwrap() + third.frame.timestamp, + advance(second.frame.timestamp, second_frames, 48_000).unwrap() ); } @@ -1351,7 +1414,7 @@ mod tests { let frame = consumer.read().await.unwrap().expect("decoded frame"); assert!( - frame.data.len() / size_of::() < encoder.frame_size(), + frame.frame.data.len() / size_of::() < encoder.frame_size(), "pre-skip is trimmed" ); // The cold-start prior reads 100 ms, plus one 20 ms packet. @@ -1456,9 +1519,9 @@ mod tests { let first = consumer.read().await.unwrap().expect("first decoded frame"); let second = consumer.read().await.unwrap().expect("second decoded frame"); - let first_frames = first.data.len() / size_of::(); - let expected = advance(first.timestamp, first_frames, 48_000).unwrap(); - assert_eq!(second.timestamp, expected); + let first_frames = first.frame.data.len() / size_of::(); + let expected = advance(first.frame.timestamp, first_frames, 48_000).unwrap(); + assert_eq!(second.frame.timestamp, expected); } #[tokio::test] @@ -1514,8 +1577,8 @@ mod tests { let first = consumer.read().await.unwrap().expect("first decoded frame"); let _second = consumer.read().await.unwrap().expect("second decoded frame"); let resumed = consumer.read().await.unwrap().expect("resumed decoded frame"); - let first_frames = first.data.len() / size_of::(); - let resumed_frames = resumed.data.len() / size_of::(); + let first_frames = first.frame.data.len() / size_of::(); + let resumed_frames = resumed.frame.data.len() / size_of::(); assert!(first_frames < frame_size, "the first epoch trims pre-skip"); assert_eq!( resumed_frames, first_frames, @@ -1566,7 +1629,7 @@ mod tests { let frame = consumer.read().await.unwrap().expect("decoded frame"); assert_eq!( - Format::F32.as_interleaved_f32(&frame.data, 1).unwrap().as_ref(), + Format::F32.as_interleaved_f32(&frame.frame.data, 1).unwrap().as_ref(), samples ); } @@ -1622,8 +1685,8 @@ mod tests { let frame = consumer.read().await.unwrap().expect("decoded frame"); // `as_micros`, not `==`: the CMAF path carries the fmp4 timescale and // `Timestamp`'s equality is structural, so the scales would have to match too. - assert_eq!(frame.timestamp.as_micros(), 0); - let samples = Format::F32.as_interleaved_f32(&frame.data, 2).unwrap(); + assert_eq!(frame.frame.timestamp.as_micros(), 0); + let samples = Format::F32.as_interleaved_f32(&frame.frame.data, 2).unwrap(); assert_eq!(samples.len(), (960 - 312) * 2); } } diff --git a/rs/moq-audio/src/encode/capture.rs b/rs/moq-audio/src/encode/capture.rs index 39cff5d6122..8c69e185e64 100644 --- a/rs/moq-audio/src/encode/capture.rs +++ b/rs/moq-audio/src/encode/capture.rs @@ -2194,7 +2194,7 @@ mod tests { async fn read_new(&self, track: &mut Track, seen: &[u64]) -> u64 { loop { let packet = track.read().await.unwrap().expect("a published packet"); - let timestamp = u64::try_from(packet.timestamp.as_micros()).unwrap(); + let timestamp = u64::try_from(packet.frame.timestamp.as_micros()).unwrap(); if !seen.contains(×tamp) { return timestamp; } diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index d92de5745b7..cab2d23f235 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -648,7 +648,7 @@ mod tests { let mut decoded = Vec::new(); while let Some(frame) = audio.read().await.unwrap() { - let pcm = Format::F32.as_interleaved_f32(&frame.data, 1).unwrap(); + let pcm = Format::F32.as_interleaved_f32(&frame.frame.data, 1).unwrap(); decoded.extend_from_slice(&pcm); } assert_eq!(decoded.len(), frames, "terminal padding extended the source"); @@ -819,7 +819,7 @@ mod tests { producer.write(&full_frame(0)).unwrap(); let first = audio.read().await.unwrap().expect("first epoch packet"); - assert_eq!(first.data.len() / size_of::(), 960 - 312); + assert_eq!(first.frame.data.len() / size_of::(), 960 - 312); producer.reset_epoch(); producer.write(&full_frame(1_000_000)).unwrap(); @@ -827,8 +827,8 @@ mod tests { let mut resumed_frames = 0; while let Some(frame) = audio.read().await.unwrap() { - if frame.timestamp.as_micros() >= 1_000_000 { - resumed_frames += frame.data.len() / size_of::(); + if frame.frame.timestamp.as_micros() >= 1_000_000 { + resumed_frames += frame.frame.data.len() / size_of::(); } } assert!(resumed_frames > 0, "the resumed epoch still decodes"); @@ -866,8 +866,8 @@ mod tests { for index in 0..100 { producer.write(&pcm_frame(&silence, index * 20_000)).unwrap(); let consumed = consumer.read().await.unwrap().expect("one decoded frame"); - assert_eq!(producer.activity(), consumed.activity); - if consumed.activity.is_dtx() { + assert_eq!(producer.activity(), consumed.frame.activity); + if consumed.frame.activity.is_dtx() { entered_dtx = true; break; } @@ -883,7 +883,7 @@ mod tests { producer.write(&pcm_frame(&active, 2_000_000)).unwrap(); let consumed = consumer.read().await.unwrap().expect("one decoded frame"); assert_eq!(producer.activity(), Activity::Active); - assert_eq!(consumed.activity, Activity::Active); + assert_eq!(consumed.frame.activity, Activity::Active); } // One 20 ms Opus frame at 48 kHz mono is exactly 960 f32 samples, so each @@ -929,7 +929,7 @@ mod tests { } producer.write(frame).unwrap(); let read = reader.read().await.unwrap().expect("a packet per full frame"); - pts.push(read.timestamp.as_micros()); + pts.push(read.frame.timestamp.as_micros()); } pts } @@ -1074,7 +1074,16 @@ mod tests { producer.write(&pcm_frame(&[0.1; 4 * 1024], epoch_us)).unwrap(); let mut pts = Vec::new(); for _ in 0..4 { - pts.push(reader.read().await.unwrap().expect("a packet").timestamp.as_micros()); + pts.push( + reader + .read() + .await + .unwrap() + .expect("a packet") + .frame + .timestamp + .as_micros(), + ); } pts } @@ -1124,7 +1133,7 @@ mod tests { producer.write(&pcm_frame(&vec![0.1; 44_100], 1_000_000)).unwrap(); let first = reader.read().await.unwrap().expect("a packet"); - assert_eq!(first.timestamp.as_micros(), 1_000_000); + assert_eq!(first.frame.timestamp.as_micros(), 1_000_000); } #[tokio::test] diff --git a/rs/moq-audio/src/playback.rs b/rs/moq-audio/src/playback.rs index 798bb5984dc..884a06a48cf 100644 --- a/rs/moq-audio/src/playback.rs +++ b/rs/moq-audio/src/playback.rs @@ -18,7 +18,7 @@ //! let mut sink = engine.sink(input)?; //! //! while let Some(frame) = audio.read().await? { -//! let write = sink.write(&frame.data)?; +//! let write = sink.write(&frame.frame.data)?; //! if write.dropped_sample_frames > 0 { //! eprintln!("dropped {} live audio frames", write.dropped_sample_frames); //! } diff --git a/rs/moq-audio/tests/import.rs b/rs/moq-audio/tests/import.rs index a1a5a147e43..160f75faeee 100644 --- a/rs/moq-audio/tests/import.rs +++ b/rs/moq-audio/tests/import.rs @@ -39,7 +39,7 @@ async fn ts_surround_opus_decodes() { .expect("decoded frame timed out") .unwrap() { - let (samples, rest) = frame.data.as_chunks::<4>(); + let (samples, rest) = frame.frame.data.as_chunks::<4>(); assert!(rest.is_empty()); for chunk in samples.as_chunks::<6>().0 { for (channel, sample) in energy.iter_mut().zip(chunk) { diff --git a/rs/moq-audio/tests/roundtrip.rs b/rs/moq-audio/tests/roundtrip.rs index 34ba2750638..af63857020c 100644 --- a/rs/moq-audio/tests/roundtrip.rs +++ b/rs/moq-audio/tests/roundtrip.rs @@ -76,6 +76,7 @@ async fn opus_round_trip_48k_stereo() { .unwrap() { let pcm: Vec = frame + .frame .data .as_chunks::<4>() .0 @@ -151,7 +152,7 @@ async fn opus_round_trip_44100_s16_resampled() { .expect("decoded frame timed out") .unwrap() { - total_bytes += frame.data.len() as u64; + total_bytes += frame.frame.data.len() as u64; } assert!( total_bytes > 10_000, @@ -192,8 +193,9 @@ async fn pcm_round_trip_is_lossless() { producer.finish().unwrap(); let frame = consumer.read().await.unwrap().unwrap(); - assert_eq!(frame.timestamp, Timestamp::from_micros(123_000).unwrap()); + assert_eq!(frame.frame.timestamp, Timestamp::from_micros(123_000).unwrap()); let decoded: Vec = frame + .frame .data .as_chunks::<4>() .0 @@ -259,6 +261,7 @@ async fn pcm_five_one_reads_back_at_any_layout() { assert_eq!(consumer.layout(), layout); let decoded = consumer.read().await.unwrap().unwrap(); let samples: Vec = decoded + .frame .data .as_chunks::<4>() .0 diff --git a/rs/moq-c/src/audio.rs b/rs/moq-c/src/audio.rs index f13b3991ba0..5bd108953ba 100644 --- a/rs/moq-c/src/audio.rs +++ b/rs/moq-c/src/audio.rs @@ -272,7 +272,7 @@ impl Audio { }; // Hold the lock only to buffer the frame; release it before the callback. - let frame_id = State::lock().audio.frames.insert(frame)?; + let frame_id = State::lock().audio.frames.insert(frame.frame)?; callback.call(Ok(frame_id)); } } diff --git a/rs/moq-c/src/consume.rs b/rs/moq-c/src/consume.rs index aed2e64588e..114c0aaae14 100644 --- a/rs/moq-c/src/consume.rs +++ b/rs/moq-c/src/consume.rs @@ -540,7 +540,7 @@ impl Consume { }; // Hold the lock only to buffer the frame; release it before the callback. - let frame_id = State::lock().consume.frame.insert(frame)?; + let frame_id = State::lock().consume.frame.insert(frame.frame)?; callback.call(Ok(frame_id)); } } diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index e66f1d88cd4..3e6f3fb9eb4 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -407,7 +407,7 @@ async fn play_audio( loop { let frame = match consumer.read().await { - Ok(Some(frame)) => frame, + Ok(Some(sample)) => sample.frame, Ok(None) => break, // One bad packet is that packet's problem: the decoder stays usable, so // skip it rather than ending playback and taking the video window down diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index e80f837b795..e42366f8c60 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -84,7 +84,8 @@ impl Video { self.presentation.lock().unwrap().video_restarted(); } let ended = frame.is_none(); - if let Some(frame) = frame { + if let Some(sample) = frame { + let frame = sample.frame; if self .presentation .lock() diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 446684f6806..f732db988d8 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -1133,6 +1133,7 @@ pub(crate) mod tests { .await .unwrap() .expect("a video frame") + .frame .timestamp; assert!( first.as_micros() < Duration::from_secs(5).as_micros(), @@ -1294,6 +1295,6 @@ pub(crate) mod tests { .expect("verbatim read timed out") .unwrap() .expect("a published verbatim frame"); - frame.payload.to_vec() + frame.frame.payload.to_vec() } } diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index 7ae6156f7c3..a67218d52cb 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -342,7 +342,7 @@ struct ConsumerInner { impl ConsumerInner { async fn next(&mut self) -> Result, MoqError> { - Ok(self.consumer.read().await?.map(Into::into)) + Ok(self.consumer.read().await?.map(|sample| sample.frame.into())) } } diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 5a51313c1c6..bf58007dbda 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -1096,7 +1096,11 @@ struct Media { impl Media { async fn next(&mut self) -> Result, MoqError> { - self.inner.read().await?.map(media_frame).transpose() + self.inner + .read() + .await? + .map(|sample| media_frame(sample.frame)) + .transpose() } } diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index e0f51009101..8aaef625710 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1080,7 +1080,7 @@ impl Pump { ); gst::event::Segment::new(&segment) }); - let buffer = build_buffer(frame, &mut reference_ts, kind); + let buffer = build_buffer(frame.frame, &mut reference_ts, kind); // pad.push() blocks until downstream accepts the buffer (full queues, a // clock-synced sink). block_in_place hands our sibling tasks to another // worker so a stalled downstream can't pin a runtime thread and starve diff --git a/rs/moq-mux/src/codec/opus/import.rs b/rs/moq-mux/src/codec/opus/import.rs index 3ee70a185a7..ed290b67831 100644 --- a/rs/moq-mux/src/codec/opus/import.rs +++ b/rs/moq-mux/src/codec/opus/import.rs @@ -178,8 +178,8 @@ mod tests { .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.payload.as_ref(), payload); - assert_eq!(frame.timestamp, Timestamp::from_micros(1_000).unwrap()); + assert_eq!(frame.frame.payload.as_ref(), payload); + assert_eq!(frame.frame.timestamp, Timestamp::from_micros(1_000).unwrap()); import.finish().unwrap(); } diff --git a/rs/moq-mux/src/codec/vp8/import.rs b/rs/moq-mux/src/codec/vp8/import.rs index dde79d1f314..f8cb4ce667b 100644 --- a/rs/moq-mux/src/codec/vp8/import.rs +++ b/rs/moq-mux/src/codec/vp8/import.rs @@ -211,8 +211,8 @@ mod tests { .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.payload, keyframe); - assert_eq!(frame.timestamp, Timestamp::from_micros(0).unwrap()); + assert_eq!(frame.frame.payload, keyframe); + assert_eq!(frame.frame.timestamp, Timestamp::from_micros(0).unwrap()); import.finish().unwrap(); } diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index bf6f4fe2d78..4ab0c6b922b 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -3,7 +3,7 @@ use std::task::{Poll, ready}; use moq_net::Timestamp; -use super::{Container, Frame, TimestampRewind}; +use super::{Container, Frame, Sample, TimestampRewind}; /// Media and clean group boundaries in delivery order. pub(crate) enum Event { @@ -94,6 +94,7 @@ pub struct Consumer { // Increments on a declared marker, an unproven delivered hole, and a latency skip. discontinuity: u64, + delivered: Option, // Increments on a declared marker only: the publisher paused or broke its timeline forward. markers: u64, @@ -143,6 +144,7 @@ impl Consumer { floor: None, presented_end: None, discontinuity: 0, + delivered: None, markers: 0, end: None, } @@ -177,7 +179,9 @@ impl Consumer { /// configured max delay. /// /// Returns `None` when the track has ended. - pub async fn read(&mut self) -> Result, F::Error> { + /// The first sample and the first one after a playhead event have `continuous: false`. + /// An undeclared forward timestamp jump alone does not change the flag. + pub async fn read(&mut self) -> Result, F::Error> { kio::wait(|waiter| self.poll_read(waiter)).await } @@ -185,10 +189,14 @@ impl Consumer { /// /// Uses a single waiter that gets registered on all relevant kio channels, /// avoiding the need for `tokio::select!` or `FuturesUnordered`. - pub fn poll_read(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { + pub fn poll_read(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { loop { match ready!(self.poll_event(waiter))? { - Some(Event::Frame(frame)) => return Poll::Ready(Ok(Some(frame))), + Some(Event::Frame(frame)) => { + let continuous = self.delivered == Some(self.discontinuity); + self.delivered = Some(self.discontinuity); + return Poll::Ready(Ok(Some(Sample { frame, continuous }))); + } Some(Event::FrameEnd(end)) => { if self.format.kind() == super::Kind::Audio { self.end = Some(end); @@ -822,7 +830,7 @@ mod tests { let mut frames = Vec::new(); loop { match tokio::time::timeout(Duration::from_millis(200), consumer.read()).await { - Ok(Ok(Some(frame))) => frames.push(frame), + Ok(Ok(Some(frame))) => frames.push(frame.frame), Ok(Ok(None)) => break, Ok(Err(e)) => return Err(e), Err(_) => panic!( @@ -903,6 +911,38 @@ mod tests { group.finish().unwrap(); } + #[tokio::test] + async fn continuity_marks_subscribe_holes_and_markers() { + let mut track = track_producer("continuity", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscription = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); + write_group(&mut track, 0, &[ts(0), ts(10_000)]); + write_group(&mut track, 2, &[ts(1_000_000), ts(1_010_000)]); + write_marker_group(&mut track, 3, ts(1_010_000)); + write_group(&mut track, 4, &[ts(2_000_000), ts(2_010_000)]); + track.finish().unwrap(); + let mut flags = Vec::new(); + while let Some(sample) = reader.read().await.unwrap() { + flags.push(sample.continuous); + } + assert_eq!(flags, [false, true, false, true, false, true]); + } + + #[tokio::test] + async fn continuity_marks_a_latency_skip() { + let mut track = track_producer("continuity", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscription = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); + reader.max_delay = Duration::ZERO; + write_group(&mut track, 0, &[ts(0)]); + write_group(&mut track, 3, &[ts(1_000_000)]); + track.finish().unwrap(); + assert!(!reader.read().await.unwrap().unwrap().continuous); + assert!(!reader.read().await.unwrap().unwrap().continuous); + } + #[tokio::test] async fn empty_group_declares_a_discontinuity() { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); @@ -915,9 +955,9 @@ mod tests { write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1); } @@ -937,11 +977,11 @@ mod tests { write_group(&mut track, 4, &[ts(2_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1, "the hole moves the playhead"); assert_eq!(consumer.markers(), 0, "but the timeline carried on"); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(2_000_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(2_000_000)); assert_eq!(consumer.discontinuity(), 2); assert_eq!(consumer.markers(), 1, "the marker declares a break"); } @@ -962,8 +1002,8 @@ mod tests { write_group(&mut track, 2, &[ts(1_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000)); assert_eq!(consumer.discontinuity(), 0, "an empty group is not a marker"); } @@ -980,9 +1020,9 @@ mod tests { write_group(&mut track, 3, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); assert_eq!( consumer.discontinuity(), 1, @@ -1172,11 +1212,11 @@ mod tests { .unwrap(); }; let read = |consumer: &mut Consumer| match consumer.poll_read(&waiter) { - Poll::Ready(Ok(Some(frame))) => Some(frame.timestamp), + Poll::Ready(Ok(Some(frame))) => Some(frame.frame.timestamp), Poll::Pending => None, other => panic!( "unexpected read: {:?}", - other.map(|r| r.map(|f| f.map(|f| f.timestamp))) + other.map(|r| r.map(|f| f.map(|f| f.frame.timestamp))) ), }; @@ -1214,7 +1254,7 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(100_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); write_group(&mut track, 1, &[ts(0)]); track.finish().unwrap(); let err = consumer.read().await.unwrap_err(); @@ -1248,10 +1288,10 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(100_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); write_group(&mut track, 1, &[ts(100_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(100_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); } /// A keyframe one frame below the previous group's last frame, but above its start, is an @@ -1264,7 +1304,7 @@ mod tests { // Read the first group before the second arrives: a blocked group that the next one // already covers is skipped for latency, which is not what this checks. for expected in [0, 33_000, 66_000] { - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(expected)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(expected)); } write_group(&mut track, 1, &[ts(50_000), ts(83_000)]); track.finish().unwrap(); @@ -1301,9 +1341,9 @@ mod tests { write_marker_group(&mut track, 1, ts(0)); write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1); } @@ -1313,11 +1353,11 @@ mod tests { let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Audio)); consumer.max_delay = Duration::ZERO; write_group(&mut track, 0, &[ts(0)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); write_marker_group(&mut track, 1, ts(0)); write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); assert_eq!( consumer.discontinuity(), 1, @@ -1343,12 +1383,12 @@ mod tests { }], ) .unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); write_marker_group(&mut track, 1, ts(10_000)); write_group(&mut track, 2, &[ts(10_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(10_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(10_000)); assert_eq!( consumer.discontinuity(), 1, @@ -1362,10 +1402,10 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(100_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); write_group(&mut track, 1, &[ts(200_000), ts(50_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(200_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(200_000)); let err = consumer.read().await.unwrap_err(); assert!(matches!( err, @@ -1451,7 +1491,7 @@ mod tests { group.finish().unwrap(); track.finish().unwrap(); - let frame = consumer.read().await.unwrap().unwrap(); + let frame = consumer.read().await.unwrap().unwrap().frame; assert!(frame.keyframe, "the marker does not consume the first-media slot"); assert!(frame.duration.is_none(), "a leading marker has no previous frame"); } @@ -1638,7 +1678,7 @@ mod tests { let result = tokio::time::timeout(Duration::from_millis(500), async { let mut frames = Vec::new(); while let Ok(Some(frame)) = consumer.read().await { - frames.push(frame); + frames.push(frame.frame); } frames }) @@ -1749,7 +1789,7 @@ mod tests { route.requested_broadcast().await.unwrap().accept(&source); track.demand().used().await.unwrap(); if sequence == 1 { - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); } for sequence in 1..=sequence { write_group(&mut track, sequence, &[ts(sequence * 1_000_000)]); @@ -1777,7 +1817,7 @@ mod tests { matches!(consumer.poll_event(&waiter), Poll::Ready(Ok(Some(Event::GroupEnd)))), "the complete successor still emits its clean boundary" ); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(2_000_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(2_000_000)); } // ---- Eviction recovery (pause/resume) ---- @@ -1807,7 +1847,7 @@ mod tests { }], ) .unwrap(); - let first = consumer.read().await.unwrap().unwrap(); + let first = consumer.read().await.unwrap().unwrap().frame; assert_eq!(first.timestamp, ts(0)); // A live group arrives far ahead -- sequences 1..4 never come (evicted). The @@ -1824,7 +1864,11 @@ mod tests { .expect("consumer hung on an evicted group / gap") .unwrap() .unwrap(); - assert_eq!(next.timestamp, ts(150_000), "skipped the evicted gap to the live group"); + assert_eq!( + next.frame.timestamp, + ts(150_000), + "skipped the evicted gap to the live group" + ); } /// A missing (evicted) sequence with a newer group buffered must be skipped once the @@ -1850,7 +1894,7 @@ mod tests { // Reading must reach group 2 across the gap instead of waiting forever for 1. let reached = tokio::time::timeout(Duration::from_secs(1), async { loop { - let frame = consumer.read().await.unwrap().unwrap(); + let frame = consumer.read().await.unwrap().unwrap().frame; if frame.timestamp == ts(200_000) { return; } @@ -1877,7 +1921,7 @@ mod tests { write_group(&mut track, 2, &[ts(200_000)]); let first = consumer.read().await.unwrap().expect("group 0 frame"); - assert_eq!(first.timestamp, ts(0)); + assert_eq!(first.frame.timestamp, ts(0)); // Group 0 is done; group 1's stream is still racing. Within the budget the // consumer waits at the gap rather than skipping to group 2. @@ -2139,7 +2183,7 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } frames }) @@ -2179,7 +2223,7 @@ mod tests { write_group(&mut track, 0, &[ts(0)]); track.finish().unwrap(); - let frame = consumer.read().await.unwrap().unwrap(); + let frame = consumer.read().await.unwrap().unwrap().frame; assert_eq!(frame.timestamp, ts(0)); consumer.set_max_delay(Duration::from_millis(100)); @@ -2223,7 +2267,7 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } frames }) @@ -2284,7 +2328,7 @@ mod tests { let _frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } frames }) @@ -2434,7 +2478,7 @@ mod tests { .expect("an unfloored join must not wait out a gap below the served start") .unwrap() .expect("track still live"); - assert_eq!(frame.timestamp, ts(3_000_000)); + assert_eq!(frame.frame.timestamp, ts(3_000_000)); } #[tokio::test] @@ -2613,7 +2657,7 @@ mod tests { track.finish().unwrap(); // Take the first frame only, leaving the second unread. - let first = consumer.read().await.unwrap().unwrap(); + let first = consumer.read().await.unwrap().unwrap().frame; assert_eq!(first.timestamp, ts(0)); // Expiry aborts the finished group while the reader is still inside it. @@ -2689,7 +2733,7 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } frames }) @@ -2746,7 +2790,7 @@ mod tests { let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } assert_eq!(frames.len(), 3); @@ -2788,7 +2832,7 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } frames }) @@ -2824,7 +2868,7 @@ mod tests { track.finish().unwrap(); let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } assert_eq!(frames.len(), 2); assert_eq!(consumer.discontinuity(), 0); @@ -2863,7 +2907,7 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } frames }) @@ -2898,7 +2942,7 @@ mod tests { .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.timestamp, ts(0)); + assert_eq!(frame.frame.timestamp, ts(0)); write_marker(&mut group, ts(15_000)); let event = kio::wait(|waiter| consumer.poll_event(waiter)).await.unwrap(); assert!(matches!(event, Some(Event::FrameEnd(end)) if end == ts(15_000))); diff --git a/rs/moq-mux/src/container/flv/import_test.rs b/rs/moq-mux/src/container/flv/import_test.rs index 4c92efcf755..4619fcbd84c 100644 --- a/rs/moq-mux/src/container/flv/import_test.rs +++ b/rs/moq-mux/src/container/flv/import_test.rs @@ -182,9 +182,12 @@ async fn import_emits_frames() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let frame = decoder.read().await.unwrap().expect("a video frame"); - assert!(frame.keyframe); + assert!(frame.frame.keyframe); // The payload is the length-prefixed NALU, carried through verbatim. - assert_eq!(frame.payload.as_ref(), &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00]); + assert_eq!( + frame.frame.payload.as_ref(), + &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00] + ); } #[tokio::test(start_paused = true)] @@ -212,7 +215,10 @@ async fn public_container_preserves_loc_for_flv() { .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.payload.as_ref(), &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00]); + assert_eq!( + frame.frame.payload.as_ref(), + &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00] + ); } /// Bytes split across two `decode` calls still reassemble into whole tags. @@ -505,8 +511,8 @@ async fn import_enhanced_av1() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let frame = decoder.read().await.unwrap().expect("an AV1 frame"); - assert!(frame.keyframe); - assert_eq!(frame.payload.as_ref(), payload); + assert!(frame.frame.keyframe); + assert_eq!(frame.frame.payload.as_ref(), payload); } /// GStreamer 1.28 `fdkaacenc` output at 48 kHz stereo, remuxed to FLV by ffmpeg 9.0.1, whose @@ -639,7 +645,7 @@ async fn import_reports_negative_pts_and_can_resume() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let frame = decoder.read().await.unwrap().expect("the good frame"); - assert_eq!(frame.timestamp.as_millis(), 10); + assert_eq!(frame.frame.timestamp.as_millis(), 10); } #[tokio::test(start_paused = true)] @@ -679,7 +685,7 @@ async fn import_enhanced_hvc1_applies_composition_time() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let frame = decoder.read().await.unwrap().expect("a video frame"); - assert_eq!(frame.timestamp.as_millis(), 17); + assert_eq!(frame.frame.timestamp.as_millis(), 17); } #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/container/group.rs b/rs/moq-mux/src/container/group.rs index 92c756af347..fc764235f8a 100644 --- a/rs/moq-mux/src/container/group.rs +++ b/rs/moq-mux/src/container/group.rs @@ -140,9 +140,9 @@ mod tests { Hang::new(&config, crate::container::Kind::Data).unwrap(), ); let first = live.read().await.unwrap().unwrap(); - assert!(first.payload.is_empty()); - assert!(first.keyframe); - assert_eq!(live.read().await.unwrap().unwrap().payload, b"data".as_slice()); + assert!(first.frame.payload.is_empty()); + assert!(first.frame.keyframe); + assert_eq!(live.read().await.unwrap().unwrap().frame.payload, b"data".as_slice()); assert!(live.read().await.unwrap().is_none()); let group = consumer.track("data").unwrap().fetch_group(0, None).await.unwrap(); let mut fetched = GroupConsumer::new(group, Hang::new(&config, crate::container::Kind::Data).unwrap()); diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index 370d8b6a11d..9fad3104ce0 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -268,7 +268,7 @@ async fn public_container_preserves_loc_for_mkv() { .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.payload.as_ref(), payload); + assert_eq!(frame.frame.payload.as_ref(), payload); } #[test] diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 0cc44b14c75..2d0bd24bb5d 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -44,6 +44,15 @@ pub enum Kind { Data, } +/// A delivered frame and whether it continues the previous delivery. +#[derive(Clone, Debug)] +pub struct Sample { + /// The encoded or decoded media frame. + pub frame: T, + /// False on the first frame and after a playhead discontinuity. + pub continuous: bool, +} + /// A decoded media frame: timestamp, payload bytes, keyframe flag. /// /// `payload` is the raw codec bitstream that gets handed to the decoder. diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index 65be578fda4..78793f4ba51 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -1683,7 +1683,7 @@ mod tests { crate::container::Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Video)); let mut timestamps = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - timestamps.push(frame.timestamp.as_micros()); + timestamps.push(frame.frame.timestamp.as_micros()); } assert_eq!(timestamps, [1_000_000, 1_040_000, 1_050_000]); } diff --git a/rs/moq-mux/src/container/test_util.rs b/rs/moq-mux/src/container/test_util.rs index f577875ed45..18853cc3bcb 100644 --- a/rs/moq-mux/src/container/test_util.rs +++ b/rs/moq-mux/src/container/test_util.rs @@ -156,7 +156,7 @@ pub(crate) async fn published( .expect("the importer finished its tracks") .unwrap() { - timestamps.push(frame.timestamp.as_micros()); + timestamps.push(frame.frame.timestamp.as_micros()); } out.insert(name, timestamps); } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index d77ca6e7041..4ea573cf296 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -1299,7 +1299,7 @@ async fn export_scte35_roundtrip() { .unwrap() .expect("no SCTE-35 frame after round-trip"); assert_eq!( - frame.payload.as_ref(), + frame.frame.payload.as_ref(), CUE, "SCTE-35 section did not round-trip byte-for-byte" ); @@ -1415,7 +1415,7 @@ async fn export_pes_verbatim_roundtrip() { .unwrap() .expect("no verbatim PES frame after round-trip"); assert_eq!( - frame.payload.as_ref(), + frame.frame.payload.as_ref(), PAYLOAD, "verbatim PES payload round-trips byte-for-byte" ); @@ -1487,7 +1487,7 @@ async fn read_frames(consumer: &moq_net::broadcast::Consumer, name: &str, kind: let mut frames = Vec::new(); while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; - frames.push(frame.payload.to_vec()); + frames.push(frame.frame.payload.to_vec()); } frames } @@ -2068,7 +2068,7 @@ async fn read_cues(consumer: &moq_net::broadcast::Consumer, name: &str) -> Vec<( let mut cues = Vec::new(); while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; - cues.push((frame.payload.to_vec(), frame.timestamp)); + cues.push((frame.frame.payload.to_vec(), frame.frame.timestamp)); } cues } diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 56b0e02534f..63c4e53e4d8 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -3729,7 +3729,7 @@ pub(super) mod test { .unwrap() .expect("a published cue frame"); assert_eq!( - &frame.payload[..], + &frame.frame.payload[..], &CUE[..], "verbatim splice_info_section after upgrade" ); @@ -4393,7 +4393,7 @@ pub(super) mod test { ); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { - frames.push(frame); + frames.push(frame.frame); } frames } @@ -4447,7 +4447,7 @@ pub(super) mod test { ); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { - frames.push(frame); + frames.push(frame.frame); } frames } @@ -5666,12 +5666,12 @@ pub(super) mod test { .unwrap() .expect("a published cue frame"); - assert_eq!(&frame.payload[..], &CUE[..], "verbatim splice_info_section"); - assert_ne!(frame.timestamp, Timestamp::ZERO, "cue must not stamp zero"); + assert_eq!(&frame.frame.payload[..], &CUE[..], "verbatim splice_info_section"); + assert_ne!(frame.frame.timestamp, Timestamp::ZERO, "cue must not stamp zero"); // The legacy container normalizes the wire timestamp to microseconds, so compare the // instant (not the raw scale) against the 90 kHz media clock the cue was stamped with. assert_eq!( - Duration::from(frame.timestamp), + Duration::from(frame.frame.timestamp), Duration::from(clock), "cue stamped with the video media clock" ); @@ -5851,7 +5851,11 @@ pub(super) mod test { .expect("verbatim read timed out") .unwrap() .expect("a published verbatim frame"); - assert_eq!(&frame.payload[..], &payload[..], "verbatim PES payload round-trips"); + assert_eq!( + &frame.frame.payload[..], + &payload[..], + "verbatim PES payload round-trips" + ); } /// A program of only verbatim PIDs reserves no rendition, so the held reservation alone keeps @@ -5941,7 +5945,7 @@ pub(super) mod test { let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { - frames.push(frame); + frames.push(frame.frame); } (frames, reader.discontinuity()) } @@ -6548,8 +6552,8 @@ pub(super) mod test { let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy(kind)); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { - if !frame.payload.is_empty() { - frames.push(frame); + if !frame.frame.payload.is_empty() { + frames.push(frame.frame); } } frames diff --git a/rs/moq-mux/src/container/ts/import_test.rs b/rs/moq-mux/src/container/ts/import_test.rs index 8b3f0662b39..bf8eb27b512 100644 --- a/rs/moq-mux/src/container/ts/import_test.rs +++ b/rs/moq-mux/src/container/ts/import_test.rs @@ -101,8 +101,8 @@ async fn public_container_preserves_loc_for_ts() { .unwrap() .unwrap() .unwrap(); - assert!(frame.keyframe); - assert!(!frame.payload.is_empty()); + assert!(frame.frame.keyframe); + assert!(!frame.frame.payload.is_empty()); } /// The Kyrion capture is H.264 1080i with B-frames (open-GOP, ~5-frame reorder despite only @@ -264,7 +264,7 @@ async fn import_opus_frames() { let mut frames = Vec::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; - frames.push(frame.payload); + frames.push(frame.frame.payload); } assert!(frames.len() > 5, "expected a run of Opus packets, got {}", frames.len()); @@ -545,7 +545,10 @@ async fn survives_midstream_join() { frames.push(frame); } assert_eq!(frames.len(), 1, "expected only the post-join IDR, got {}", frames.len()); - assert!(frames[0].keyframe, "the first surviving frame must be the keyframe"); + assert!( + frames[0].frame.keyframe, + "the first surviving frame must be the keyframe" + ); } /// A real Ateme Kyrion broadcast captured mid-stream with `nc`, so it opens dirty: @@ -596,7 +599,7 @@ async fn kyrion_dirtystart_extracts_real_cues() { ); let mut cues = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { - cues.push((frame.payload.to_vec(), frame.timestamp)); + cues.push((frame.frame.payload.to_vec(), frame.frame.timestamp)); } assert_eq!(cues.len(), 6, "expected the six real splice_inserts"); assert!( diff --git a/rs/moq-mux/src/container/ts/programs.rs b/rs/moq-mux/src/container/ts/programs.rs index 291c8c49f00..c6c59b4ad51 100644 --- a/rs/moq-mux/src/container/ts/programs.rs +++ b/rs/moq-mux/src/container/ts/programs.rs @@ -380,12 +380,15 @@ mod test { let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let container = Container::try_from(config).unwrap(); let frame = Consumer::new(track, container).read().await.unwrap().expect("a frame"); - assert!(fills.contains(&frame.payload[4]), "{path} carries only its own program"); + assert!( + fills.contains(&frame.frame.payload[4]), + "{path} carries only its own program" + ); let skew = Duration::from_secs(2); let wall = catalog .clock .expect("the catalog advertises a clock") - .wall_clock(frame.timestamp) + .wall_clock(frame.frame.timestamp) .unwrap(); assert!( before - skew <= wall && wall <= after + skew, diff --git a/rs/moq-mux/src/import/track.rs b/rs/moq-mux/src/import/track.rs index 9aca70feae6..208b58d8fed 100644 --- a/rs/moq-mux/src/import/track.rs +++ b/rs/moq-mux/src/import/track.rs @@ -1055,8 +1055,8 @@ mod tests { .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.payload, payload); - assert_eq!(frame.timestamp, Timestamp::from_micros(1_000).unwrap()); + assert_eq!(frame.frame.payload, payload); + assert_eq!(frame.frame.timestamp, Timestamp::from_micros(1_000).unwrap()); import.finish().unwrap(); } @@ -1383,9 +1383,9 @@ mod tests { .unwrap() .unwrap() .unwrap(); - assert!(frame.keyframe, "an IDR access unit must open a group"); - assert_eq!(frame.payload, au, "the payload must pass through verbatim"); - assert_eq!(frame.timestamp, Timestamp::from_micros(1_000).unwrap()); + assert!(frame.frame.keyframe, "an IDR access unit must open a group"); + assert_eq!(frame.frame.payload, au, "the payload must pass through verbatim"); + assert_eq!(frame.frame.timestamp, Timestamp::from_micros(1_000).unwrap()); import.finish().unwrap(); } diff --git a/rs/moq-rtc/src/codec/mod.rs b/rs/moq-rtc/src/codec/mod.rs index f83ca447faf..5b3a6c4f0ff 100644 --- a/rs/moq-rtc/src/codec/mod.rs +++ b/rs/moq-rtc/src/codec/mod.rs @@ -234,13 +234,13 @@ impl Track { return Ok(None); }; let payload = match &self.convert { - TrackConvert::Passthrough => frame.payload, + TrackConvert::Passthrough => frame.frame.payload, TrackConvert::LengthPrefixed { length_size, keyframe_prefix, } => { - let prefix = frame.keyframe.then(|| keyframe_prefix.as_ref()); - moq_mux::codec::annexb::from_length_prefixed(&frame.payload, *length_size, prefix) + let prefix = frame.frame.keyframe.then(|| keyframe_prefix.as_ref()); + moq_mux::codec::annexb::from_length_prefixed(&frame.frame.payload, *length_size, prefix) .map_err(moq_mux::Error::from)? } }; @@ -248,7 +248,7 @@ impl Track { continue; } return Ok(Some(PacketizedFrame { - timestamp_us: frame.timestamp.as_micros() as u64, + timestamp_us: frame.frame.timestamp.as_micros() as u64, payload, })); } diff --git a/rs/moq-srt/src/ts.rs b/rs/moq-srt/src/ts.rs index b3fdf755bbd..8c376692453 100644 --- a/rs/moq-srt/src/ts.rs +++ b/rs/moq-srt/src/ts.rs @@ -634,7 +634,7 @@ mod tests { .expect("cue read timed out") .unwrap() .expect("a published cue section"); - let expected_cue = cue.payload; + let expected_cue = cue.frame.payload; assert_eq!(expected_cue[0], 0xFC, "a verbatim splice_info_section (table_id 0xFC)"); let mut subscriber = Subscriber::new(&origin.consume(), "ingest", Options::default()) @@ -683,9 +683,12 @@ mod tests { .expect("round-trip cue read timed out") .unwrap() .expect("SRT egress preserves a cue section"); - assert_eq!(cue.payload[0], 0xFC, "the round-trip cue is a splice_info_section"); assert_eq!( - cue.payload, expected_cue, + cue.frame.payload[0], 0xFC, + "the round-trip cue is a splice_info_section" + ); + assert_eq!( + cue.frame.payload, expected_cue, "SRT egress preserves the complete cue section" ); } diff --git a/rs/moq-video/src/capture/wgc.rs b/rs/moq-video/src/capture/wgc.rs index ce0c7ede8c9..d426fc0b6fb 100644 --- a/rs/moq-video/src/capture/wgc.rs +++ b/rs/moq-video/src/capture/wgc.rs @@ -328,7 +328,7 @@ mod tests { let frame = crate::Frame::new(crate::Surface::I420(surface.clone()), timestamp); producer.publish(&encoder.encode(&frame).unwrap()).unwrap(); let received = subscriber.read().await.unwrap().unwrap(); - assert_eq!(received.timestamp.as_micros(), u128::from(step) * 20_000); + assert_eq!(received.frame.timestamp.as_micros(), u128::from(step) * 20_000); } } diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 7704dd365b7..8e4259ad055 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -194,7 +194,11 @@ impl Consumer { self.pending.extend( self.decoder - .decode(mux_frame.payload, mux_frame.timestamp, mux_frame.keyframe) + .decode( + mux_frame.frame.payload, + mux_frame.frame.timestamp, + mux_frame.frame.keyframe, + ) .await?, ); } diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index 732930d15aa..1d2b5a27efc 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -597,13 +597,13 @@ mod tests { let mut track = moq_mux::container::Consumer::new(track, moq_mux::catalog::hang::Container::try_from(config).unwrap()); let sample = track.read().await.unwrap().expect("an imported sample"); - assert!(sample.keyframe); + assert!(sample.frame.keyframe); let decoder = super::Decoder::new(config, &decode_config(super::Kind::Named(probe::BUFFERED_NAME.into()))).unwrap(); let access_unit = decoder .conversion - .access_unit(&sample.payload, sample.keyframe) + .access_unit(&sample.frame.payload, sample.frame.keyframe) .unwrap(); let mut buf = access_unit.clone(); let mut nals = moq_mux::codec::annexb::NalIterator::new(&mut buf); diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 44be26c103a..de8a7aa3227 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -1209,7 +1209,7 @@ mod tests { async fn read(track: &mut moq_mux::container::Consumer) -> u64 { let frame = track.read().await.unwrap().expect("a published frame"); - u64::try_from(frame.timestamp.as_micros()).unwrap() + u64::try_from(frame.frame.timestamp.as_micros()).unwrap() } /// Read the next frame not already in `seen`: a resubscription replays retained groups first. From bf8973101a7e136804797a2200bd0bc84f38d7e2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 17:30:47 -0700 Subject: [PATCH 03/10] fix(audio): preserve continuity across resampler tails Co-Authored-By: GPT-6 Astra --- quest/m1/rust-continuous.md | 16 ++++- rs/moq-audio/src/decode/consumer.rs | 92 +++++++++++++++++++++++++---- 2 files changed, 95 insertions(+), 13 deletions(-) diff --git a/quest/m1/rust-continuous.md b/quest/m1/rust-continuous.md index c1448c53401..7bb0604a03f 100644 --- a/quest/m1/rust-continuous.md +++ b/quest/m1/rust-continuous.md @@ -41,7 +41,21 @@ picture and loses the new picture boundary, so that approach is not included. Carry input continuity through codec output association before extending the video read result. Prefer private decoder metadata association over a timestamp heuristic, and cover delayed output on both sides of a marker. -The video propagation goal remains open here. +The video propagation goal remains open here. Native backends need explicit +input identity, not a map keyed only by presentation time: equal timestamps +can straddle a marker. VideoToolbox and Media Foundation already map synthetic +sample times, OpenH264 queues presentation times, and VAAPI, NVDEC and +MediaCodec carry codec timestamps. Media Foundation also has an unmatched-output +fallback. Decide how a backend reports an output with no surviving input +identity before exposing continuity: recommend an explicit error, rather than +borrowing the latest input's flag. Keep dropped-input bookkeeping bounded and +cover delayed output, duplicate timestamps and decoder recovery on each backend. + +The audio path now drains a resampler's buffered tail before applying a +playhead boundary. A paused-time regression previously returned flags +`false, false, true`, incorrectly joining the old tail into the new span; +it now returns the old body/tail and new body/tail as +`false, true, false, true`. ## Related diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 70f444e4755..35f9a3ad7f3 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -67,8 +67,8 @@ impl Options { /// /// The mirror of [`encode::Producer`](crate::encode::Producer): output format / /// sample rate / layout are fixed at construction, and -/// [`read`](Self::read) returns [`Frame`]s carrying the codec activity they -/// were decoded from. +/// [`read`](Self::read) returns samples carrying each frame's delivery continuity +/// and codec activity. pub struct Consumer { decoder: Decoder, track: moq_mux::container::Consumer, @@ -274,10 +274,11 @@ impl Consumer { let mux_frame = self.track.read().await?; self.apply_discontinuity()?; let Some(sample) = mux_frame else { - return Ok(self.flush()?.map(|frame| { + if let Some(frame) = self.flush()? { let continuous = std::mem::replace(&mut self.continuous, true); - Sample { frame, continuous } - })); + self.ready.push_back(Sample { frame, continuous }); + } + return Ok(self.ready.pop_front()); }; let mux_frame = sample.frame; @@ -420,6 +421,12 @@ impl Consumer { return Ok(()); } + // Samples already buffered belong to the old span, even when the next + // packet declares a new one. Drain before clearing their timeline labels. + if let Some(frame) = self.drain()? { + let continuous = std::mem::replace(&mut self.continuous, true); + self.ready.push_back(Sample { frame, continuous }); + } self.discontinuity = discontinuity; self.next_start = None; self.spans.clear(); @@ -444,13 +451,7 @@ impl Consumer { fn gap(&mut self) -> Result, Error> { self.decoder.reset_prediction()?; - let mut frame = None; - if let Some(resampler) = self.resampler.as_mut() { - let held = resampler.held_at(); - let skipped = resampler.skipped(); - let pcm = resampler.drain()?; - frame = self.tail(pcm, held, skipped)?; - } + let frame = self.drain()?; self.next_start = None; self.spans.clear(); @@ -460,6 +461,17 @@ impl Consumer { Ok(frame) } + /// Drain one span while keeping the resampler available for the next. + fn drain(&mut self) -> Result, Error> { + let Some(resampler) = self.resampler.as_mut() else { + return Ok(None); + }; + let held = resampler.held_at(); + let skipped = resampler.skipped(); + let pcm = resampler.drain()?; + self.tail(pcm, held, skipped) + } + /// The tail the resampler is still holding when the track ends, once. /// /// Without it the last partial chunk is dropped, which is up to a chunk of @@ -876,6 +888,62 @@ mod tests { assert!(consumer.read().await.unwrap().is_none()); } + #[tokio::test(start_paused = true)] + async fn resampled_continuity_keeps_the_tail_before_a_marker() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let subscriber = broadcast.consume(); + let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 44_100, 1); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + let mut consumer = Consumer::new( + &subscriber, + &catalog, + "audio", + Options { + max_delay: Duration::from_secs(10), + output: Output { + sample_rate: Some(48_000), + ..Output::default() + }, + ..Options::new() + }, + ) + .await + .unwrap(); + for index in 0..2 { + if index != 0 { + producer.discontinuity().unwrap(); + } + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_micros(index * 1_000_000).unwrap(), + duration: None, + payload: vec![0u8; 1024 * size_of::()].into(), + keyframe: true, + }) + .unwrap(); + } + producer.finish().unwrap(); + let mut samples = Vec::new(); + while let Some(sample) = consumer.read().await.unwrap() { + samples.push((sample.frame.timestamp.as_micros(), sample.continuous)); + } + assert_eq!( + samples.iter().map(|(_, continuous)| *continuous).collect::>(), + [false, true, false, true] + ); + assert!( + samples[1].0 < 1_000_000, + "the old resampler tail stays before the marker" + ); + assert_eq!(samples[2].0, 1_000_000, "the new span starts on its source timestamp"); + } + #[tokio::test] async fn resampling_keeps_the_activity_boundary_on_its_source() { let mut encoder = Encoder::new(&Settings { From 9f07a7a19a9014049c0bf8b3ce049df9c70e8fc7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 12:44:18 -0700 Subject: [PATCH 04/10] refactor(mux): name the read wrapper container::Delivery; split video continuity Rename container::Sample to container::Delivery so it does not collide with PCM or MP4 samples. Complete the Rust continuity quest for container and audio reads and move video propagation to quest/m1/video-continuous.md. Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 2 +- quest/m1/audio-warmup.md | 8 +-- quest/m1/open-gop-leading-pictures.md | 6 +-- quest/m1/rust-continuous.md | 62 ----------------------- quest/m1/video-continuous.md | 39 ++++++++++++++ quest/m2/intra-refresh/consumer-warmup.md | 11 ++-- rs/moq-audio/src/decode/consumer.rs | 32 ++++++------ rs/moq-cli/src/play/media.rs | 2 +- rs/moq-cli/src/play/video.rs | 4 +- rs/moq-ffi/src/audio.rs | 2 +- rs/moq-ffi/src/media.rs | 2 +- rs/moq-mux/src/container/consumer.rs | 14 ++--- rs/moq-mux/src/container/mod.rs | 2 +- 13 files changed, 79 insertions(+), 107 deletions(-) delete mode 100644 quest/m1/rust-continuous.md create mode 100644 quest/m1/video-continuous.md diff --git a/quest/m1/README.md b/quest/m1/README.md index f3ade6dbc7d..409e9398c4b 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -166,7 +166,7 @@ folded into Dropped sources. - [Encoder colour](/quest/m1/color-model.md) - every moq-video encode path signals the colour its output actually has, or refuses instead of mislabelling - [T-STD TS export](/quest/m1/tstd/README.md) - `moq export ts` is a proper remux that passes the T-STD buffer model, starting with a fixed `--delay` - [ATSC AC-3](/quest/m1/ts-atsc-ac3.md) - `export ts` carries 48 kHz ATSC AC-3 at every A/52 rate up to 640 kb/s instead of aborting from 384 kb/s up -- [Rust non-continuous signal](/quest/m1/rust-continuous.md) - the Rust container consumer reports a frame after a subscribe or discontinuity as non-continuous, like JS, for the tune-in and warmup trims +- [Rust video delivery continuity](/quest/m1/video-continuous.md) - Rust video decode reads report each picture's delivery continuity, like the container and audio reads, for the tune-in and warmup trims - [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - a viewer joining at a recovery point drops the leading pictures it cannot decode; continuous viewers keep them - [Audio warmup](/quest/m1/audio-warmup.md) - a viewer joining an Opus rendition mid-stream never hears the unconverged first 80 ms - [#3021](/quest/m1/3021-moq-gst-anchor-generated-media-timelines-to-wall-clock.md) - moq-gst picks the broadcast wall epoch; a restarted source is a new epoch, not a forward re-anchor diff --git a/quest/m1/audio-warmup.md b/quest/m1/audio-warmup.md index 9160f0e82dd..02c92f3a1ac 100644 --- a/quest/m1/audio-warmup.md +++ b/quest/m1/audio-warmup.md @@ -15,8 +15,8 @@ frames decode independently and set nothing; HE-AAC is out of scope. publish `warmup` for Opus; `js/publish` does the same for its Opus track. - `rs/moq-audio/src/decode` already trims Opus `pre_skip` at stream start; the warmup trim is the same mechanism keyed on the container consumer's - non-continuous signal (added in Rust by - [Rust non-continuous signal](/quest/m1/rust-continuous.md)), and the + non-continuous signal (`continuous` on the Rust audio decoder's + `container::Delivery`), and the subscription's maximum age grows by `warmup` as the video consumer quest does. `js/watch` audio mirrors it. - The trim replaces `LEGACY_WARMUP_CALLBACKS` in `js/watch/src/audio/decoder.ts`, @@ -26,10 +26,6 @@ frames decode independently and set nothing; HE-AAC is out of scope. - Tests in both languages: a mid-stream join discards exactly the warmup span and a continuous listener loses nothing. -## Required - -- [Rust non-continuous signal](/quest/m1/rust-continuous.md) - the signal the trim keys on - ## Related - [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - trims video at tune-in from the same signal diff --git a/quest/m1/open-gop-leading-pictures.md b/quest/m1/open-gop-leading-pictures.md index e6fb70d2bee..4ad437f5261 100644 --- a/quest/m1/open-gop-leading-pictures.md +++ b/quest/m1/open-gop-leading-pictures.md @@ -50,8 +50,8 @@ decode, and the JS test should drive the software decoder. Every continuous group is passed through untouched. - The same rule in the Rust decode path (`moq-video` decode consumers), so native playback and the transcoder tune in the same way, keyed on the - non-continuous signal from [Rust non-continuous - signal](/quest/m1/rust-continuous.md). + non-continuous signal from [Rust video delivery + continuity](/quest/m1/video-continuous.md). - The same rule in `moq export ts`. Decided (2026-10-01): the fixed-delay export still sends a join's orphaned leading pictures, so 3 of 500 frames on the open-GOP fixture decode after they present @@ -75,7 +75,7 @@ audio warmup and consumer warmup don't wait on this one. ## Required -- [Rust non-continuous signal](/quest/m1/rust-continuous.md) - the Rust decode path keys the trim on it +- [Rust video delivery continuity](/quest/m1/video-continuous.md) - the Rust decode path keys the trim on it ## Related diff --git a/quest/m1/rust-continuous.md b/quest/m1/rust-continuous.md deleted file mode 100644 index 7bb0604a03f..00000000000 --- a/quest/m1/rust-continuous.md +++ /dev/null @@ -1,62 +0,0 @@ -# [S] Rust non-continuous signal - -## Goal - -`moq_mux::container::Consumer` tells its reader whether a frame picks up -exactly where the previous one left off, like `continuous` in -`js/hang/src/container/consumer.ts`: false on the first frame after the -subscribe and after every discontinuity, true otherwise. -`rs/moq-video/src/decode` and `rs/moq-audio/src/decode` pass it through, so -the open-GOP trim, audio warmup, and consumer warmup key on one signal -instead of each adding its own. - -## Plan - -Split out of [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) -in the 2026-10-08 audit, so audio and consumer warmup need not wait on the -open-GOP trim. - -Today `poll_read` returns a bare frame, and `discontinuity()` is a counter -bumped on a declared marker group, an unproven delivered hole, or a latency -skip, but not on the subscribe itself. Report the flag with each frame and -let the counter go if nothing else needs it. Mirror the JS name. - -Tests: the first frame after a subscribe, a marker group, a delivered hole, -and a latency skip each read as non-continuous, and every other frame as -continuous. - -Public API: breaking in moq-mux (the consumer's read result changes). Wire: -none. - -### Draft progress and remaining video propagation - -The draft adds a shared Sample envelope to container and audio reads, with -regressions for subscribe, markers, holes, skips, and audio decode. Existing -consumers explicitly unwrap the frame; binding surfaces stay unchanged. - -Video still returns its existing decoded Frame. A decoder can return a picture -from before a marker while processing the access unit after it, or return it -only on flush. A single pending boolean attaches the new boundary to the old -picture and loses the new picture boundary, so that approach is not included. -Carry input continuity through codec output association before extending the -video read result. Prefer private decoder metadata association over a -timestamp heuristic, and cover delayed output on both sides of a marker. -The video propagation goal remains open here. Native backends need explicit -input identity, not a map keyed only by presentation time: equal timestamps -can straddle a marker. VideoToolbox and Media Foundation already map synthetic -sample times, OpenH264 queues presentation times, and VAAPI, NVDEC and -MediaCodec carry codec timestamps. Media Foundation also has an unmatched-output -fallback. Decide how a backend reports an output with no surviving input -identity before exposing continuity: recommend an explicit error, rather than -borrowing the latest input's flag. Keep dropped-input bookkeeping bounded and -cover delayed output, duplicate timestamps and decoder recovery on each backend. - -The audio path now drains a resampler's buffered tail before applying a -playhead boundary. A paused-time regression previously returned flags -`false, false, true`, incorrectly joining the old tail into the new span; -it now returns the old body/tail and new body/tail as -`false, true, false, true`. - -## Related - -- [Consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md) - withholds frames after the same signal diff --git a/quest/m1/video-continuous.md b/quest/m1/video-continuous.md new file mode 100644 index 00000000000..2c3b47db74d --- /dev/null +++ b/quest/m1/video-continuous.md @@ -0,0 +1,39 @@ +# [M] Rust video delivery continuity + +## Goal + +`moq_video::decode::Consumer::read` returns `container::Delivery`, so +each decoded picture says whether it continues the previous delivery, like +the container and audio reads already do. A picture decoded from input before +a marker, skip, or subscribe keeps that input's flag even when the codec +returns it late, and the first picture after the boundary reads +non-continuous. + +## Plan + +Split from the container and audio work in +[#5288](https://github.com/moq-dev/moq/pull/5288), where the maintainer chose +the `container::Delivery { frame, continuous }` wrapper and landing video +separately. + +A decoder can return a picture from before a marker while processing the +access unit after it, or only on flush, so a single pending boolean attaches +the boundary to the wrong picture. Carry a private input identity through +every backend (`rs/moq-video/src/decode/backend`) and map each output back to +its input's flag. A map keyed only by presentation time is not enough: equal +timestamps can straddle a marker. VideoToolbox and Media Foundation already +map synthetic sample times, OpenH264 queues presentation times, and VAAPI, +NVDEC and MediaCodec carry codec timestamps. An output whose input identity +did not survive (Media Foundation's unmatched-output fallback) is an explicit +error, never a borrowed flag. Keep dropped-input bookkeeping bounded. + +Tests: delayed output on both sides of a marker, duplicate timestamps, and +decoder recovery, on each backend CI can run. + +Public API: breaking in moq-video (the read result). Bindings keep their +existing frame records. Wire: none. + +## Related + +- [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - the Rust decode trim keys on this signal +- [Consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md) - withholds frames after the same signal diff --git a/quest/m2/intra-refresh/consumer-warmup.md b/quest/m2/intra-refresh/consumer-warmup.md index 536638fcad3..350a6951ae3 100644 --- a/quest/m2/intra-refresh/consumer-warmup.md +++ b/quest/m2/intra-refresh/consumer-warmup.md @@ -22,11 +22,10 @@ replaces both: the rule is timestamp arithmetic on the group start. - Withhold rule, keyed on the same non-continuous signal in both consumers: `js/hang/src/container/consumer.ts` `next()` reports `continuous: false` after a subscribe, a declared discontinuity, or any skip (`#gap`). The Rust - `moq_mux::container::Consumer` gains the equivalent in - [Rust non-continuous signal](/quest/m1/rust-continuous.md) - (today `poll_read` returns a bare frame and only the `discontinuity()` - counter moves); this quest reuses it, and - `rs/moq-video/src/decode/consumer.rs` propagates it. For the first + `moq_mux::container::Consumer` reports the equivalent as + `container::Delivery::continuous`; this quest reuses it once + `rs/moq-video/src/decode/consumer.rs` propagates it in + [Rust video delivery continuity](/quest/m1/video-continuous.md). For the first group after that signal, every frame is decoded (the decoder needs them to build reference state) and frames stamped below `group.start + warmup` are not presented; frames stamped at or above that boundary are, so the recovery @@ -60,7 +59,7 @@ replaces both: the rule is timestamp arithmetic on the group start. ## Required -- [Rust non-continuous signal](/quest/m1/rust-continuous.md) - the Rust signal this keys on +- [Rust video delivery continuity](/quest/m1/video-continuous.md) - the Rust video signal this keys on ## Related diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 908e4f5d773..50abb09d301 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -1,6 +1,6 @@ //! Subscribe to an encoded audio track and emit raw PCM. -use moq_mux::container::Sample; +use moq_mux::container::Delivery; use std::collections::VecDeque; use std::time::{Duration, Instant}; @@ -67,7 +67,7 @@ impl Options { /// /// The mirror of [`encode::Producer`](crate::encode::Producer): output format / /// sample rate / layout are fixed at construction, and -/// [`read`](Self::read) returns samples carrying each frame's delivery continuity +/// [`read`](Self::read) returns deliveries carrying each frame's continuity /// and codec activity. pub struct Consumer { decoder: Decoder, @@ -85,7 +85,7 @@ pub struct Consumer { next_start: Option, /// Frames decoded and not yet handed back, so a gap's tail can be returned /// ahead of the packet that exposed it. - ready: VecDeque>, + ready: VecDeque>, /// Codec activity spans the resampler's buffered output still covers. spans: VecDeque, /// Activity of the last span the output ran past, for the rounding samples the @@ -253,7 +253,7 @@ impl Consumer { } /// Read the next decoded PCM frame, or `None` when the track ends. - /// The returned sample carries the container's delivery continuity through codec buffering. + /// The returned delivery carries the container's continuity through codec buffering. /// /// [`Frame::activity`] reports whether the packet these samples came from /// coded audio. It describes where the frame begins, so a resampled @@ -265,7 +265,7 @@ impl Consumer { /// side stay anchored to their own packet timeline, so the hole is there to /// see. "Doesn't continue" allows for the quantization the stamps carry, which /// on a millisecond-stamped ingest is most of a millisecond. - pub async fn read(&mut self) -> Result>, Error> { + pub async fn read(&mut self) -> Result>, Error> { loop { if let Some(frame) = self.ready.pop_front() { return Ok(Some(frame)); @@ -273,15 +273,15 @@ impl Consumer { let mux_frame = self.track.read().await?; self.apply_discontinuity()?; - let Some(sample) = mux_frame else { + let Some(delivery) = mux_frame else { if let Some(frame) = self.flush()? { let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Sample { frame, continuous }); + self.ready.push_back(Delivery { frame, continuous }); } return Ok(self.ready.pop_front()); }; - let mux_frame = sample.frame; + let mux_frame = delivery.frame; // Every packet the container hands over is an arrival, including one the // decoder is about to refuse, so it is observed before anything can fail. let arrival = jitter::millis(now().duration_since(self.origin)); @@ -311,10 +311,10 @@ impl Consumer { && let Some(frame) = self.gap()? { let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Sample { frame, continuous }); + self.ready.push_back(Delivery { frame, continuous }); } - self.continuous &= sample.continuous; + self.continuous &= delivery.continuous; let rate = self.decoder.sample_rate(); let epoch = *self.epoch.get_or_insert(mux_frame.timestamp); let delay = self.decoder.delay_remaining(); @@ -409,7 +409,7 @@ impl Consumer { // same iteration still comes out first. The next turn of the loop pops it. let frame = self.frame(pcm, timestamp, activity)?; let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Sample { frame, continuous }); + self.ready.push_back(Delivery { frame, continuous }); } } @@ -425,7 +425,7 @@ impl Consumer { // packet declares a new one. Drain before clearing their timeline labels. if let Some(frame) = self.drain()? { let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Sample { frame, continuous }); + self.ready.push_back(Delivery { frame, continuous }); } self.discontinuity = discontinuity; self.next_start = None; @@ -657,8 +657,8 @@ mod tests { } producer.finish().unwrap(); let mut flags = Vec::new(); - while let Some(sample) = consumer.read().await.unwrap() { - flags.push(sample.continuous); + while let Some(delivery) = consumer.read().await.unwrap() { + flags.push(delivery.continuous); } assert_eq!(flags, [false, true, false, true]); } @@ -930,8 +930,8 @@ mod tests { } producer.finish().unwrap(); let mut samples = Vec::new(); - while let Some(sample) = consumer.read().await.unwrap() { - samples.push((sample.frame.timestamp.as_micros(), sample.continuous)); + while let Some(delivery) = consumer.read().await.unwrap() { + samples.push((delivery.frame.timestamp.as_micros(), delivery.continuous)); } assert_eq!( samples.iter().map(|(_, continuous)| *continuous).collect::>(), diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index e0ff402987c..9ba35c89676 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -442,7 +442,7 @@ async fn play_audio( loop { let frame = match consumer.read().await { - Ok(Some(sample)) => sample.frame, + Ok(Some(delivery)) => delivery.frame, Ok(None) => break, // One bad packet is that packet's problem: the decoder stays usable, so // skip it rather than ending playback and taking the video window down diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index e42366f8c60..a97506f3600 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -84,8 +84,8 @@ impl Video { self.presentation.lock().unwrap().video_restarted(); } let ended = frame.is_none(); - if let Some(sample) = frame { - let frame = sample.frame; + if let Some(delivery) = frame { + let frame = delivery.frame; if self .presentation .lock() diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index 0d8d1ba3820..5d3463a85a8 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -342,7 +342,7 @@ struct ConsumerInner { impl ConsumerInner { async fn next(&mut self) -> Result, MoqError> { - Ok(self.consumer.read().await?.map(|sample| sample.frame.into())) + Ok(self.consumer.read().await?.map(|delivery| delivery.frame.into())) } } diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 88f5ee0e448..0c8fc417b17 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -1104,7 +1104,7 @@ impl Media { self.inner .read() .await? - .map(|sample| media_frame(sample.frame)) + .map(|delivery| media_frame(delivery.frame)) .transpose() } } diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 51232cdd91f..6959ca36696 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -3,7 +3,7 @@ use std::task::{Poll, ready}; use moq_net::Timestamp; -use super::{Container, Frame, Sample, TimestampRewind}; +use super::{Container, Delivery, Frame, TimestampRewind}; /// Media and clean group boundaries in delivery order. pub(crate) enum Event { @@ -179,9 +179,9 @@ impl Consumer { /// configured max delay. /// /// Returns `None` when the track has ended. - /// The first sample and the first one after a playhead event have `continuous: false`. + /// The first delivery and the first one after a playhead event have `continuous: false`. /// An undeclared forward timestamp jump alone does not change the flag. - pub async fn read(&mut self) -> Result, F::Error> { + pub async fn read(&mut self) -> Result, F::Error> { kio::wait(|waiter| self.poll_read(waiter)).await } @@ -189,13 +189,13 @@ impl Consumer { /// /// Uses a single waiter that gets registered on all relevant kio channels, /// avoiding the need for `tokio::select!` or `FuturesUnordered`. - pub fn poll_read(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { + pub fn poll_read(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { loop { match ready!(self.poll_event(waiter))? { Some(Event::Frame(frame)) => { let continuous = self.delivered == Some(self.discontinuity); self.delivered = Some(self.discontinuity); - return Poll::Ready(Ok(Some(Sample { frame, continuous }))); + return Poll::Ready(Ok(Some(Delivery { frame, continuous }))); } Some(Event::FrameEnd(end)) => { if self.format.kind() == super::Kind::Audio { @@ -928,8 +928,8 @@ mod tests { write_group(&mut track, 4, &[ts(2_000_000), ts(2_010_000)]); track.finish().unwrap(); let mut flags = Vec::new(); - while let Some(sample) = reader.read().await.unwrap() { - flags.push(sample.continuous); + while let Some(delivery) = reader.read().await.unwrap() { + flags.push(delivery.continuous); } assert_eq!(flags, [false, true, false, true, false, true]); } diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 76890fd8a33..77670d47115 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -47,7 +47,7 @@ pub enum Kind { /// A delivered frame and whether it continues the previous delivery. #[derive(Clone, Debug)] -pub struct Sample { +pub struct Delivery { /// The encoded or decoded media frame. pub frame: T, /// False on the first frame and after a playhead discontinuity. From 5a0e640119e68fc0fe29dcba4bbaf9288728192a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 13:16:13 -0700 Subject: [PATCH 05/10] fix(mux): mark the delivery after an evicted current group non-continuous An aborted current group skipped to the next buffered group without advancing the playhead, so the next frame read as continuous although the rest of the evicted group was lost. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/consumer.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 6959ca36696..42f4703bc86 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -53,8 +53,9 @@ pub(crate) enum Event { /// /// A marker group (no decodable frames, one empty payload) is a walk-now discontinuity. /// A delivered sequence hole is a playhead event unless the boundary is contiguous within -/// 1 ms, and a latency skip is the same event. [`discontinuity`](Self::discontinuity) is -/// that playhead generation: re-apply startup delay and skip, not a decoder flush. +/// 1 ms, and a latency skip or an evicted current group is the same event. +/// [`discontinuity`](Self::discontinuity) is that playhead generation: re-apply startup +/// delay and skip, not a decoder flush. /// Empty groups (zero objects) mean nothing. A group whose timestamps fall below the live /// edge earlier groups reached is malformed and aborts the track. pub struct Consumer { @@ -151,7 +152,7 @@ impl Consumer { } /// A counter that increments at each playhead event: a declared marker group, an - /// unproven delivered hole, or a latency skip. + /// unproven delivered hole, a latency skip, or an evicted current group. /// /// Downstream consumers re-apply startup delay and skip when it changes. It is not a /// decoder flush: the next group already starts on a keyframe with parameter sets. @@ -307,6 +308,8 @@ impl Consumer { error = ?e, "current group evicted; skipping to next buffered group" ); + // Whatever the group still held is lost, so this is a skip like any other. + self.bump_playhead(); self.pending.pop_front(); self.current = self.pending.front().map_or(self.current + 1, |g| g.sequence); continue 'read; @@ -1874,6 +1877,7 @@ mod tests { ts(150_000), "skipped the evicted gap to the live group" ); + assert!(!next.continuous, "the eviction skip starts a new span"); } /// A missing (evicted) sequence with a newer group buffered must be skipped once the From 48ec8c687ced34f055b0103531b9178b66be871f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 19:42:21 -0700 Subject: [PATCH 06/10] feat(mux): read gaps and declared ends as items in delivery order Replace the container::Delivery wrapper and the polled discontinuity()/end() accessors with one read stream in both languages: - Rust: container::Read::{Frame { group, frame }, Gap, End { group, end }} - JS: Consumer.next() yields { kind: "frame", group, frame } | { kind: "gap" } | { kind: "end", group, end }; the continuous/discontinuity fields, the group-done results, and the discontinuity getter are gone. A gap is media this reader will never get (the join, a skip, an eviction, a truncation, an unproven hole). An end is the publisher declaring its media stops: a marker group or an audio terminal. The frames that follow an end from its own group are the stopped encoder's drain; the next group is fresh. Timestamps cannot tell those apart (drain packets are stamped past the end), so the group rides along, as the hang draft scopes the endpoint. Trailing video duration markers stay internal. moq-cli play now applies a gap or end before the frame after it instead of polling a counter after the read. Drops quest/m1/video-continuous.md: decoders read the gap directly. Co-Authored-By: Claude Opus 5.5 --- doc/lib/rs/moq-mux.md | 6 +- .../container/consumer.nonsequential.test.ts | 15 +- .../src/container/consumer.outoforder.test.ts | 53 +-- js/hang/src/container/consumer.rewind.test.ts | 3 +- js/hang/src/container/consumer.skips.test.ts | 3 +- .../src/container/consumer.startup.test.ts | 66 ++-- js/hang/src/container/consumer.test.ts | 283 ++++++-------- js/hang/src/container/consumer.ts | 147 +++---- js/hang/src/container/index.ts | 2 +- js/watch/src/audio/decoder.test.ts | 14 +- js/watch/src/audio/decoder.ts | 28 +- js/watch/src/audio/playout.test.ts | 2 +- js/watch/src/audio/replay.ts | 4 +- js/watch/src/audio/terminal.test.ts | 51 ++- js/watch/src/audio/terminal.ts | 53 +-- js/watch/src/retention.test.ts | 6 +- js/watch/src/video/decoder.test.ts | 46 +-- js/watch/src/video/decoder.ts | 66 ++-- quest/m1/README.md | 1 - quest/m1/audio-warmup.md | 6 +- quest/m1/open-gop-leading-pictures.md | 27 +- quest/m1/video-continuous.md | 39 -- quest/m2/intra-refresh/consumer-warmup.md | 18 +- rs/hang/examples/subscribe.rs | 18 +- rs/moq-audio/src/decode/consumer.rs | 244 +++++------- rs/moq-audio/src/encode/capture.rs | 7 +- rs/moq-audio/src/encode/producer.rs | 53 +-- rs/moq-audio/src/playback.rs | 2 +- rs/moq-audio/tests/import.rs | 2 +- rs/moq-audio/tests/roundtrip.rs | 7 +- rs/moq-c/src/audio.rs | 2 +- rs/moq-c/src/consume.rs | 8 +- rs/moq-cli/src/play/media.rs | 2 +- rs/moq-cli/src/play/video.rs | 47 +-- rs/moq-cli/src/publish.rs | 29 +- rs/moq-ffi/src/audio.rs | 2 +- rs/moq-ffi/src/media.rs | 13 +- rs/moq-gst/src/source/imp.rs | 5 +- rs/moq-mux/src/codec/opus/import.rs | 6 +- rs/moq-mux/src/codec/vp8/import.rs | 6 +- rs/moq-mux/src/container/consumer.rs | 363 +++++++++++------- rs/moq-mux/src/container/flv/import_test.rs | 30 +- rs/moq-mux/src/container/fmp4/export.rs | 2 +- rs/moq-mux/src/container/fmp4/import_test.rs | 6 +- rs/moq-mux/src/container/group.rs | 10 +- rs/moq-mux/src/container/mkv/import_test.rs | 14 +- rs/moq-mux/src/container/mod.rs | 27 +- rs/moq-mux/src/container/producer.rs | 4 +- rs/moq-mux/src/container/source.rs | 2 +- rs/moq-mux/src/container/test_util.rs | 4 +- rs/moq-mux/src/container/ts/export_test.rs | 16 +- rs/moq-mux/src/container/ts/import.rs | 41 +- rs/moq-mux/src/container/ts/import_test.rs | 25 +- rs/moq-mux/src/container/ts/programs.rs | 13 +- rs/moq-mux/src/import/track.rs | 14 +- rs/moq-rtc/src/codec/mod.rs | 15 +- rs/moq-srt/src/ts.rs | 25 +- rs/moq-video/src/capture/wgc.rs | 8 +- rs/moq-video/src/decode/consumer.rs | 25 +- rs/moq-video/src/decode/decoder.rs | 11 +- rs/moq-video/src/encode/producer.rs | 8 +- 61 files changed, 1028 insertions(+), 1027 deletions(-) delete mode 100644 quest/m1/video-continuous.md diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index de9782a4f65..82255ac22ce 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -30,9 +30,9 @@ set at the init segment, so a new rendition or a changed configuration ends the export. The [CLI page](/bin/cli#import) covers these and the MPEG-TS specifics: one program per broadcast, damaged packets, and feed checks. -Container and audio reads return a frame with its delivery continuity: the first -frame and the first after a skip, hole, or declared break start a new span. -Video decoders return bare frames. +A container read yields frames plus two markers in delivery order: a gap where +media will never arrive (the join, a skip, a hole) and an end where the +publisher declared its media stops (a pause, a source switch, the finish). Data tracks are catalog entries too. The [hang page](/concept/hang#data-tracks) describes the modes. A capture time converts onto the broadcast clock; a diff --git a/js/hang/src/container/consumer.nonsequential.test.ts b/js/hang/src/container/consumer.nonsequential.test.ts index fca4c3f4a76..61a62ea97a9 100644 --- a/js/hang/src/container/consumer.nonsequential.test.ts +++ b/js/hang/src/container/consumer.nonsequential.test.ts @@ -5,6 +5,14 @@ import { Format as LegacyFormat } from "./legacy.ts"; // Standalone from consumer.test.ts so it only pulls the Legacy format, not the CMAF decoder. +/** The next frame, passing over gaps and ends. */ +async function nextFrame(consumer: Consumer) { + for (;;) { + const next = await consumer.next(); + if (!next || next.kind === "frame") return next; + } +} + function encodeLegacyFrame(timestamp: Time.Micro, payload: Uint8Array): Uint8Array { const tsBytes = Varint.encode(timestamp); const data = new Uint8Array(tsBytes.byteLength + payload.byteLength); @@ -40,10 +48,9 @@ test("Consumer waits on a non-sequential hang group because it has no per-frame a.close(); track.writeGroup(a); - // Drain A: its frame, then its group-done marker. - const firstFrame = await consumer.next(); - expect(firstFrame?.frame?.payload).toEqual(new Uint8Array([0x01])); - await consumer.next(); + // Drain A's frame. + const firstFrame = await nextFrame(consumer); + expect(firstFrame?.frame.payload).toEqual(new Uint8Array([0x01])); // Park next() before the next group's frames arrive (the live streaming case). const pending = consumer.next(); diff --git a/js/hang/src/container/consumer.outoforder.test.ts b/js/hang/src/container/consumer.outoforder.test.ts index 89453f3d1f6..2bccec79ca7 100644 --- a/js/hang/src/container/consumer.outoforder.test.ts +++ b/js/hang/src/container/consumer.outoforder.test.ts @@ -5,6 +5,14 @@ import { Format as LegacyFormat } from "./legacy.ts"; // Standalone from consumer.test.ts so it only pulls the Legacy format, not the CMAF decoder. +/** The next frame, passing over gaps and ends. */ +async function nextFrame(consumer: Consumer) { + for (;;) { + const next = await consumer.next(); + if (!next || next.kind === "frame") return next; + } +} + function encodeLegacyFrame(timestamp: Time.Micro, payload: Uint8Array): Uint8Array { const tsBytes = Varint.encode(timestamp); const data = new Uint8Array(tsBytes.byteLength + payload.byteLength); @@ -29,14 +37,16 @@ function publish(track: Track.Producer, sequence: number, millis: number) { group.close(); } -/** Every group and frame the consumer will hand over right now, as [group, payload] pairs. */ +/** Every frame the consumer will hand over right now, as [group, payload] pairs. */ async function drain(consumer: Consumer): Promise<[number, number | undefined][]> { const seen: [number, number | undefined][] = []; for (;;) { - const next = await Promise.race([consumer.next(), new Promise((resolve) => setTimeout(resolve, 50))]); + const next = await Promise.race([ + consumer.next(), + new Promise((resolve) => setTimeout(resolve, 50)), + ]); if (!next) break; - const { group, frame } = next as { group: number; frame?: { payload: Uint8Array } }; - seen.push([group, frame?.payload[0]]); + if (next.kind === "frame") seen.push([next.group, next.frame.payload[0]]); } return seen; } @@ -55,7 +65,7 @@ test("out-of-order groups are delivered rather than dropped", async () => { // The live edge lands first, and delivery starts there rather than waiting. publish(track, 3, 30); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([3])); + expect((await nextFrame(consumer))?.frame.payload).toEqual(new Uint8Array([3])); // The window's head follows it. Every group is still within both budgets, so all three // are handed over even though each sits below the group already delivered. @@ -63,8 +73,7 @@ test("out-of-order groups are delivered rather than dropped", async () => { publish(track, 1, 10); publish(track, 2, 20); - const delivered = (await drain(consumer)).filter(([, payload]) => payload !== undefined); - expect(delivered).toEqual([ + expect(await drain(consumer)).toEqual([ [0, 0], [1, 1], [2, 2], @@ -87,25 +96,25 @@ test("a below-cursor group still downloading is not truncated", async () => { const live = new Group.Producer(3); track.writeGroup(live); publishFrame(live, 3000, 30); - expect((await consumer.next())?.frame?.payload[0]).toBe(30); + expect((await nextFrame(consumer))?.frame.payload[0]).toBe(30); // A backlog group lands below the cursor, one frame at a time (mid-download). const backlog = new Group.Producer(1); track.writeGroup(backlog); publishFrame(backlog, 1000, 10); - const first = await consumer.next(); + const first = await nextFrame(consumer); expect(first?.group).toBe(1); - expect(first?.frame?.payload[0]).toBe(10); + expect(first?.frame.payload[0]).toBe(10); // The consumer finds the group's buffer empty while it is still open. It must wait // rather than shift the group out, so the second frame still arrives. - const pending = consumer.next(); + const pending = nextFrame(consumer); publishFrame(backlog, 1500, 11); backlog.close(); - const second = (await pending) as { group: number; frame?: { payload: Uint8Array } }; - expect(second.group).toBe(1); - expect(second.frame?.payload[0]).toBe(11); + const second = await pending; + expect(second?.group).toBe(1); + expect(second?.frame.payload[0]).toBe(11); live.close(); track.close(); @@ -120,30 +129,24 @@ test("delayed older groups survive a floor established by two newer groups", asy format: new LegacyFormat("data"), maxDelay: Time.Milli(5000), }); - async function nextFrame() { - for (;;) { - const next = await consumer.next(); - if (!next || next.frame) return next; - } - } try { publish(track, 3, 30); - expect((await nextFrame())?.group).toBe(3); + expect((await nextFrame(consumer))?.group).toBe(3); const latest = new Group.Producer(4); track.writeGroup(latest); publishFrame(latest, 40, 4); - expect((await nextFrame())?.group).toBe(4); + expect((await nextFrame(consumer))?.group).toBe(4); // Group 1's 10ms frame is valid history, below group 4's 30ms floor. publish(track, 1, 10); - const backlog = await nextFrame(); + const backlog = await nextFrame(consumer); expect(backlog?.group).toBe(1); - expect(backlog?.frame?.timestamp).toBe(Time.Micro(10_000)); + expect(backlog?.frame.timestamp).toBe(Time.Micro(10_000)); // Delivering the backlog must not erase the floor for group 4 itself. publishFrame(latest, 29, 5); - await expect(nextFrame()).rejects.toThrow("below the previous group start"); + await expect(nextFrame(consumer)).rejects.toThrow("below the previous group start"); } finally { consumer.close(); track.close(); diff --git a/js/hang/src/container/consumer.rewind.test.ts b/js/hang/src/container/consumer.rewind.test.ts index c826fbe1b58..34adadab02e 100644 --- a/js/hang/src/container/consumer.rewind.test.ts +++ b/js/hang/src/container/consumer.rewind.test.ts @@ -38,7 +38,7 @@ function group(track: Track.Producer, timestamps: number[], end?: number) { async function frame(consumer: Consumer) { for (;;) { const result = await consumer.next(); - if (!result || result.frame) return result; + if (!result || result.kind === "frame") return result; } } @@ -50,7 +50,6 @@ test("Consumer accepts a resuming keyframe before the estimated group end", asyn group(track, [0, 33_000], 66_000); expect((await frame(consumer))?.frame?.timestamp).toBe(Time.Micro(0)); expect((await frame(consumer))?.frame?.timestamp).toBe(Time.Micro(33_000)); - expect((await consumer.next())?.end).toBe(Time.Micro(66_000)); group(track, [50_000]); expect((await frame(consumer))?.frame?.timestamp).toBe(Time.Micro(50_000)); } finally { diff --git a/js/hang/src/container/consumer.skips.test.ts b/js/hang/src/container/consumer.skips.test.ts index 591ab00aa1b..17447495cf3 100644 --- a/js/hang/src/container/consumer.skips.test.ts +++ b/js/hang/src/container/consumer.skips.test.ts @@ -34,7 +34,8 @@ for (const count of [1, 16, 256]) { expect(warn).toHaveBeenLastCalledWith(`skipping slow groups: track=tone 0 -> ${count * 2} count=${count}`); expect(consumer.buffered.peek()[0]?.start).toBe(Time.Milli(count)); // The retained group is still delivered, and another catch-up reports separately. - expect((await consumer.next())?.group).toBe(count * 2); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: count * 2 }); newest.writeFrame({ payload: Varint.encode(count * 1000), timestamp: Time.Timestamp.now() }); write(count * 2 + 2, (count + 1) * 1000); await buffered(count + 1); diff --git a/js/hang/src/container/consumer.startup.test.ts b/js/hang/src/container/consumer.startup.test.ts index 008116a911f..5d8ecfacd8f 100644 --- a/js/hang/src/container/consumer.startup.test.ts +++ b/js/hang/src/container/consumer.startup.test.ts @@ -1,6 +1,6 @@ import { expect, jest, onTestFinished, spyOn, test } from "bun:test"; import { Group, Error as NetError, StreamCode, Time, Track } from "@moq/net"; -import { Consumer } from "./consumer.ts"; +import { Consumer, type Read } from "./consumer.ts"; import { encodeFrame, Format } from "./legacy.ts"; // Drain the in-memory readers and their continuations without advancing mocked time. @@ -13,7 +13,7 @@ function setup(maxDelay = 100) { format: new Format("video"), maxDelay: Time.Milli(maxDelay), }); - const results: NonNullable>>[] = []; + const results: Read[] = []; const warn = spyOn(console, "warn").mockImplementation(() => {}); const reading = (async () => { for (;;) { @@ -35,6 +35,19 @@ function setup(maxDelay = 100) { return { track, consumer, results, warn }; } +// Each item as "gap", "@<µs>" for a frame, or "end @<µs>". +function items(results: Read[]): string[] { + return results.map((read) => { + if (read.kind === "gap") return "gap"; + if (read.kind === "end") return `end ${read.group}@${read.end}`; + return `${read.group}@${read.frame.timestamp}`; + }); +} + +function frames(results: Read[]) { + return results.flatMap((read) => (read.kind === "frame" ? [read] : [])); +} + function write(group: Group.Producer, timestamp: number, marker = false) { group.writeFrame({ payload: encodeFrame(marker ? new Uint8Array() : new Uint8Array([1]), Time.Micro(timestamp)), @@ -46,7 +59,7 @@ for (const emptySuccessor of [false, true]) { test.each([0, 100])( `Consumer starts across a hole after an empty reset (empty successor: ${emptySuccessor}, maxDelay: %s)`, async (maxDelay) => { - const { track, consumer, results, warn } = setup(maxDelay); + const { track, results, warn } = setup(maxDelay); const stale = new Group.Producer(16); track.writeGroup(stale); await flush(); @@ -58,8 +71,7 @@ for (const emptySuccessor of [false, true]) { } stale.close(new NetError.Stream(StreamCode.Old)); await flush(); - expect(results.some((result) => result.group === 16 && !result.frame)).toBe(true); - if (emptySuccessor) expect(results.some((result) => result.group === 17 && !result.frame)).toBe(true); + expect(results).toEqual([]); const sequence = emptySuccessor ? 19 : 18; const live = new Group.Producer(sequence); @@ -67,36 +79,34 @@ for (const emptySuccessor of [false, true]) { for (let i = 0; i < 5; i++) write(live, 1_000_000 + i * 33_333); await flush(); - const frames = results.filter((result) => result.frame); - expect(frames.map((result) => result.group)).toEqual(Array(5).fill(sequence)); - expect(frames.map((result) => result.frame?.timestamp)).toEqual( - [1_000_000, 1_033_333, 1_066_666, 1_099_999, 1_133_332].map((timestamp) => Time.Micro(timestamp)), - ); - expect(frames.map((result) => result.frame?.keyframe)).toEqual([true, false, false, false, false]); - expect(frames.map((result) => result.continuous)).toEqual([false, true, true, true, true]); - expect(consumer.discontinuity).toBe(0); + expect(items(results)).toEqual([ + "gap", + ...[1_000_000, 1_033_333, 1_066_666, 1_099_999, 1_133_332].map( + (timestamp) => `${sequence}@${timestamp}`, + ), + ]); + expect(frames(results).map((read) => read.frame.keyframe)).toEqual([true, false, false, false, false]); expect(warn).not.toHaveBeenCalled(); }, ); } test("Consumer preserves a declared marker before its first media across a hole", async () => { - const { track, consumer, results, warn } = setup(); + const { track, results, warn } = setup(); const marker = new Group.Producer(16); track.writeGroup(marker); write(marker, 0, true); marker.close(); await flush(); - expect(consumer.discontinuity).toBe(1); + expect(items(results)).toEqual(["gap", "end 16@0"]); const live = new Group.Producer(18); track.writeGroup(live); write(live, 50_000); await flush(); - const frames = results.filter((result) => result.frame); - expect(frames).toHaveLength(1); - expect(frames[0]).toMatchObject({ group: 18, discontinuity: 1, continuous: false }); - expect(frames[0].frame?.keyframe).toBe(true); + // Group 17 never arrived, so the media after the marker is past a gap. + expect(items(results)).toEqual(["gap", "end 16@0", "gap", "18@50000"]); + expect(frames(results)[0].frame.keyframe).toBe(true); expect(warn).not.toHaveBeenCalled(); }); @@ -107,7 +117,7 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(first, 0); first.close(); await flush(); - expect(results.filter((result) => result.frame)).toHaveLength(1); + expect(items(results)).toEqual(["gap", "16@0"]); if (reset) { const marker = new Group.Producer(17); @@ -115,7 +125,7 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(marker, 0, true); marker.close(); await flush(); - expect(consumer.discontinuity).toBe(1); + expect(items(results)).toEqual(["gap", "16@0", "end 17@0"]); } const sequence = reset ? 20 : 19; const later = new Group.Producer(sequence); @@ -123,7 +133,7 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(later, 50_000); await flush(); expect(consumer.buffered.peek().length).toBeGreaterThan(0); - expect(results.filter((result) => result.frame)).toHaveLength(1); + expect(frames(results)).toHaveLength(1); // Supply the missing, timestamp-contiguous bridge and prove the queued frame survives. const bridge = new Group.Producer(sequence - 1); @@ -131,11 +141,13 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(bridge, 0); bridge.close(); await flush(); - expect(results.filter((result) => result.frame).map((result) => result.group)).toEqual([ - 16, - sequence - 1, - sequence, + // The bridge closes the hole, so nothing reads as a gap past the first. + expect(items(results)).toEqual([ + "gap", + "16@0", + ...(reset ? ["end 17@0"] : []), + `${sequence - 1}@0`, + `${sequence}@50000`, ]); - expect(consumer.discontinuity).toBe(reset ? 1 : 0); expect(warn).not.toHaveBeenCalled(); }); diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 0443a095d38..a2c740a4002 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -5,7 +5,7 @@ import { AudioConfigSchema } from "../catalog/audio.ts"; import { decodeInitSegment, type InitSegment } from "./cmaf/decode.ts"; import { createAudioInitSegment, encodeDataSegment } from "./cmaf/encode.ts"; import { Format as CmafFormat } from "./cmaf/format.ts"; -import { Consumer } from "./consumer.ts"; +import { Consumer, type Read } from "./consumer.ts"; import type { Format as ContainerFormat } from "./format.ts"; import { Format as LegacyFormat, Producer as LegacyProducer } from "./legacy.ts"; import type { Frame } from "./types.ts"; @@ -522,13 +522,27 @@ async function drainFrames( new Promise((resolve) => setTimeout(() => resolve(null), timeout)), ]); if (result === null || result === undefined) break; - if (result.frame) { + if (result.kind === "frame") { frames.push({ timestamp: result.frame.timestamp, group: result.group, keyframe: result.frame.keyframe }); } } return frames; } +/** An item as "gap", a frame's µs, or "end <µs>". */ +function describe(read: Read): string { + if (read.kind === "gap") return "gap"; + if (read.kind === "end") return `end ${read.end}`; + return `${read.frame.timestamp}`; +} + +/** The frame a read carries, failing on anything else. */ +function frameOf(read: Read | string | undefined): Frame { + if (typeof read !== "object" || read.kind !== "frame") + throw new Error(`expected a frame, got ${JSON.stringify(read)}`); + return read.frame; +} + test("Consumer delivers frames from a single group", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Time.Milli }); @@ -731,7 +745,7 @@ test("Consumer delivers a group that arrives below the cursor", async () => { // --- Group.Producer boundary signals --- -test("Consumer next() returns group-done signals", async () => { +test("Consumer next() yields a gap before the first frame only", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Time.Milli }); @@ -741,26 +755,21 @@ test("Consumer next() returns group-done signals", async () => { await new Promise((resolve) => setTimeout(resolve, 50)); - const allResults: { frame: boolean; group: number }[] = []; + const seen: string[] = []; for (;;) { const result = await Promise.race([ consumer.next(), new Promise((resolve) => setTimeout(() => resolve(null), 500)), ]); if (result === null || result === undefined) break; - allResults.push({ frame: result.frame !== undefined, group: result.group }); + seen.push(describe(result)); } - const frameResults = allResults.filter((r) => r.frame); - const boundaries = allResults.filter((r) => !r.frame); - expect(frameResults).toHaveLength(3); - expect(boundaries).toHaveLength(2); - expect(boundaries[0].group).toBe(0); - expect(boundaries[1].group).toBe(1); + expect(seen).toEqual(["gap", "0", "33000", "66000"]); consumer.close(); }); -test("Consumer reports a duration marker as metadata", async () => { +test("Consumer keeps a trailing duration marker internal", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("video"), @@ -781,18 +790,17 @@ test("Consumer reports a duration marker as metadata", async () => { track.close(); await settle(); - const media = await consumer.next(); - expect(media?.frame?.payload).toEqual(new Uint8Array([0xde, 0xad])); - expect(media?.frame?.keyframe).toBe(true); - expect(media?.frame?.duration).toBeUndefined(); - - const done = await consumer.next(); - expect(done?.frame).toBeUndefined(); - expect(done?.end).toBe(33_000 as Time.Micro); + expect(await consumer.next()).toEqual({ kind: "gap" }); + const media = frameOf(await consumer.next()); + expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); + expect(media.keyframe).toBe(true); + expect(media.duration).toBeUndefined(); + // The marker is the frame's duration, not an item. + expect(await consumer.next()).toBeUndefined(); consumer.close(); }); -test("Consumer skips a leading marker and keeps the first media keyframe", async () => { +test("Consumer reads a leading marker as an end and keeps the first media keyframe", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("video"), @@ -812,17 +820,16 @@ test("Consumer skips a leading marker and keeps the first media keyframe", async track.writeGroup(group); track.close(); - const marker = await consumer.next(); - expect(marker?.end).toBe(20_000 as Time.Micro); - expect(marker?.frame).toBeUndefined(); - const media = await consumer.next(); - expect(media?.frame?.payload).toEqual(new Uint8Array([0xde, 0xad])); - expect(media?.frame?.keyframe).toBe(true); - expect(media?.frame?.duration).toBeUndefined(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(await consumer.next()).toEqual({ kind: "end", group: 0, end: 20_000 as Time.Micro }); + const media = frameOf(await consumer.next()); + expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); + expect(media.keyframe).toBe(true); + expect(media.duration).toBeUndefined(); consumer.close(); }); -test("Consumer skips an empty LOC payload", async () => { +test("Consumer reads an empty LOC payload as the frame's duration", async () => { const track = new Track.Producer("test"); const producer = new LocProducer(track); producer.encode(new Uint8Array([0xde, 0xad]), 0 as Time.Micro, true); @@ -831,16 +838,15 @@ test("Consumer skips an empty LOC payload", async () => { const consumer = new Consumer(replay(track), { format: new LocFormat("video"), maxDelay: 500 as Time.Milli }); await settle(); - const media = await consumer.next(); - expect(media?.frame?.payload).toEqual(new Uint8Array([0xde, 0xad])); - expect(media?.frame?.duration).toBeUndefined(); - const marker = await consumer.next(); - expect(marker?.end).toBe(33_000 as Time.Micro); - expect(marker?.frame).toBeUndefined(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + const media = frameOf(await consumer.next()); + expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); + expect(media.duration).toBeUndefined(); + expect(await consumer.next()).toBeUndefined(); consumer.close(); }); -test("Consumer skips the duration marker the LOC producer writes", async () => { +test("Consumer passes over the duration markers the LOC producer writes", async () => { const track = new Track.Producer("test"); const producer = new LocProducer(track); producer.encode(new Uint8Array([0xde, 0xad]), 0 as Time.Micro, true); @@ -850,23 +856,15 @@ test("Consumer skips the duration marker the LOC producer writes", async () => { const consumer = new Consumer(replay(track), { format: new LocFormat("video"), maxDelay: 500 as Time.Milli }); await settle(); - const first = await consumer.next(); - expect(first?.frame?.payload).toEqual(new Uint8Array([0xde, 0xad])); - expect(first?.frame?.keyframe).toBe(true); - const second = await consumer.next(); - expect(second?.frame?.payload).toEqual(new Uint8Array([0xbe, 0xef])); - const boundary = await consumer.next(); - expect(boundary?.frame).toBeUndefined(); - expect(boundary?.end).toBe(20_000 as Time.Micro); - const keyframe = await consumer.next(); - expect(keyframe?.frame).toBeUndefined(); - expect(keyframe?.end).toBeUndefined(); - const next = await consumer.next(); - expect(next?.frame?.payload).toEqual(new Uint8Array([0xca, 0xfe])); - expect(next?.frame?.keyframe).toBe(true); - const tail = await consumer.next(); - expect(tail?.frame).toBeUndefined(); - expect(tail?.end).toBe(30_000 as Time.Micro); + expect(await consumer.next()).toEqual({ kind: "gap" }); + const first = frameOf(await consumer.next()); + expect(first.payload).toEqual(new Uint8Array([0xde, 0xad])); + expect(first.keyframe).toBe(true); + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0xbe, 0xef])); + const next = frameOf(await consumer.next()); + expect(next.payload).toEqual(new Uint8Array([0xca, 0xfe])); + expect(next.keyframe).toBe(true); + expect(await consumer.next()).toBeUndefined(); consumer.close(); }); @@ -879,21 +877,21 @@ test("Consumer preserves empty Legacy and LOC data frames", async () => { producer.close(); const format = kind === "legacy" ? new LegacyFormat("data") : new LocFormat(); const consumer = new Consumer(replay(track), { format, maxDelay: 500 as Time.Milli }); - const result = await consumer.next(); - expect(result?.frame?.payload).toEqual(new Uint8Array()); - expect(result?.frame?.keyframe).toBe(true); - expect(result?.end).toBeUndefined(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + const frame = frameOf(await consumer.next()); + expect(frame.payload).toEqual(new Uint8Array()); + expect(frame.keyframe).toBe(true); consumer.close(); } }); // --- Malformed rewind --- -/** Read until the next media frame, skipping the group-done markers in between. */ +/** Read until the next media frame, passing over gaps and ends. */ async function nextFrame(consumer: Consumer) { for (;;) { const result = await consumer.next(); - if (!result || result.frame) return result; + if (!result || result.kind === "frame") return result; } } @@ -918,7 +916,6 @@ test("Consumer aborts a rewind carried by a later arrival", async () => { writeGroupWithLegacyFrames(track, 0, [10_000_000 as Time.Micro]); await settle(); expect((await nextFrame(consumer))?.frame?.timestamp).toBe(10_000_000 as Time.Micro); - expect((await consumer.next())?.frame).toBeUndefined(); const pending = nextFrame(consumer); await settle(); @@ -937,20 +934,17 @@ test("Consumer treats B-frame reordering within a group as continuous", async () writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 66_000 as Time.Micro, 33_000 as Time.Micro]); await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); for (const timestamp of [0, 66_000, 33_000]) { - const result = await nextFrame(consumer); - expect(result?.frame?.timestamp).toBe(timestamp as Time.Micro); - expect(result?.discontinuity).toBe(0); + expect(frameOf(await consumer.next()).timestamp).toBe(timestamp as Time.Micro); } // The cursor is now on group 1, whose own frames reorder the same way. writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro, 166_000 as Time.Micro, 133_000 as Time.Micro]); await settle(); for (const timestamp of [100_000, 166_000, 133_000]) { - const result = await nextFrame(consumer); - expect(result?.frame?.timestamp).toBe(timestamp as Time.Micro); - expect(result?.discontinuity).toBe(0); - expect(result?.continuous).toBe(true); + // Read directly: a gap here would fail the frame check. + expect(frameOf(await consumer.next()).timestamp).toBe(timestamp as Time.Micro); } consumer.close(); @@ -963,10 +957,9 @@ test("Consumer accepts open-GOP leading pictures above the previous group", asyn writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 33_000 as Time.Micro]); writeGroupWithLegacyFrames(track, 1, [66_000 as Time.Micro, 50_000 as Time.Micro]); await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); for (const timestamp of [0, 33_000, 66_000, 50_000]) { - const result = await nextFrame(consumer); - expect(result?.frame?.timestamp).toBe(timestamp as Time.Micro); - expect(result?.discontinuity).toBe(0); + expect(frameOf(await consumer.next()).timestamp).toBe(timestamp as Time.Micro); } consumer.close(); @@ -1061,8 +1054,7 @@ test("Consumer preserves empty media from formats without endpoint markers", asy track.writeGroup(group); track.close(); - const result = await consumer.next(); - expect(result?.frame?.payload).toHaveLength(0); + expect(frameOf(await nextFrame(consumer)).payload).toHaveLength(0); consumer.close(); }); @@ -1256,9 +1248,7 @@ test("Consumer delivers a PTS-contiguous next group whose sequence jumped (CMAF) a.close(); track.writeGroup(a); - const firstFrame = await consumer.next(); - expect(firstFrame?.frame?.payload).toEqual(new Uint8Array([0x01])); - await consumer.next(); // group-done marker + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); const pending = consumer.next(); @@ -1285,7 +1275,7 @@ test("Consumer delivers a PTS-contiguous next group whose sequence jumped (CMAF) new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 500)), ]); expect(result).not.toBe("timeout"); - expect((result as { frame?: Frame } | undefined)?.frame?.payload).toEqual(new Uint8Array([0x02])); + expect(frameOf(result).payload).toEqual(new Uint8Array([0x02])); consumer.close(); }); @@ -1317,9 +1307,7 @@ test("Consumer waits on a PTS gap instead of skipping to a later buffered group a.close(); track.writeGroup(a); - const firstFrame = await consumer.next(); - expect(firstFrame?.frame?.payload).toEqual(new Uint8Array([0x01])); - await consumer.next(); // group-done marker + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); const pending = consumer.next(); @@ -1362,7 +1350,7 @@ test("Consumer waits on a PTS gap instead of skipping to a later buffered group }); const result = await pending; - expect((result as { frame?: Frame } | undefined)?.frame?.payload).toEqual(new Uint8Array([0x02])); + expect(frameOf(result).payload).toEqual(new Uint8Array([0x02])); consumer.close(); }); @@ -1383,7 +1371,7 @@ test("Consumer delivers a contiguous group after one that completed out of order track.writeGroup(a); a.writeFrame(cmafFrame(0x01, 0, 0)); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x01])); + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); // A's second frame stays buffered so A can't be duration-skipped. A ends at 6000 ticks (66_666µs). a.writeFrame(cmafFrame(0x02, 3000, 1)); @@ -1398,10 +1386,8 @@ test("Consumer delivers a contiguous group after one that completed out of order await settle(); a.close(); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x02])); - expect((await consumer.next())?.frame).toBeUndefined(); // A group-done - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x03])); - expect((await consumer.next())?.frame).toBeUndefined(); // B group-done + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x03])); // C (seq 1_090_000) starts at B's end, so it continues the timeline and must deliver. const pending = consumer.next(); @@ -1411,7 +1397,7 @@ test("Consumer delivers a contiguous group after one that completed out of order const result = await Promise.race([pending, settle(300).then(() => "timeout" as const)]); expect(result).not.toBe("timeout"); - expect((result as { frame?: Frame } | undefined)?.frame?.payload).toEqual(new Uint8Array([0x04])); + expect(frameOf(result).payload).toEqual(new Uint8Array([0x04])); consumer.close(); }); @@ -1430,9 +1416,8 @@ test("Consumer plays the head once a waited-out gap exceeds the budget (CMAF)", const a = new Group.Producer(1000); track.writeGroup(a); a.writeFrame(cmafFrame(0x01, 0, 0)); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x01])); + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); a.close(); - expect((await consumer.next())?.frame).toBeUndefined(); // #active falls back to the 1001 phantom // The reader is parked on the missing 1001, as a decoder is while it waits for the next group. const pending = consumer.next(); @@ -1443,12 +1428,10 @@ test("Consumer plays the head once a waited-out gap exceeds the budget (CMAF)", track.writeGroup(b); b.writeFrame(cmafFrame(0x02, 90_000, 1)); + // The missing group's span was never delivered, so a gap comes first. const result = await Promise.race([pending, settle(300).then(() => "timeout" as const)]); - expect(result).not.toBe("timeout"); - const delivered = result as { frame?: Frame; continuous?: boolean } | undefined; - expect(delivered?.frame?.payload).toEqual(new Uint8Array([0x02])); - // The missing group's span was never delivered, so downstream must not bridge it. - expect(delivered?.continuous).toBe(false); + expect(result).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); consumer.close(); }); @@ -1465,9 +1448,8 @@ test("Consumer waits on a gap within the budget, then plays the head once it exc const a = new Group.Producer(1000); track.writeGroup(a); a.writeFrame(cmafFrame(0x01, 0, 0)); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x01])); + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); a.close(); - expect((await consumer.next())?.frame).toBeUndefined(); // B (seq 2000) starts at 6000 ticks (66_667µs): a gap, but only 33ms past A's end. const b = new Group.Producer(2000); @@ -1485,36 +1467,36 @@ test("Consumer waits on a gap within the budget, then plays the head once it exc } const result = await Promise.race([pending, settle(300).then(() => "timeout" as const)]); - expect(result).not.toBe("timeout"); - expect((result as { frame?: Frame } | undefined)?.frame?.payload).toEqual(new Uint8Array([0x02])); + expect(result).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); consumer.close(); }); -// --- Continuity reporting --- +// --- Gaps and ends --- -// `continuous` is what downstream buffer accounting keys on, so it must be false exactly when the -// consumer dropped something. Group numbers can't answer that: they aren't required to be -// sequential, and adjacency doesn't rule out a group the max delay check dropped on the way past. -test("Consumer reports continuity while nothing is dropped", async () => { +// A gap is what downstream buffer accounting keys on, so it must appear exactly when the consumer +// dropped something. Group numbers can't answer that: they aren't required to be sequential, and +// adjacency doesn't rule out a group the max delay check dropped on the way past. +test("Consumer yields no gap while nothing is dropped", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 100 as Time.Milli }); writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 33_000 as Time.Micro]); await settle(); - expect((await consumer.next())?.continuous).toBe(false); // nothing precedes the first frame - expect((await consumer.next())?.continuous).toBe(true); // same group: consecutive by protocol - expect((await consumer.next())?.frame).toBeUndefined(); // group 0 done + expect(await consumer.next()).toEqual({ kind: "gap" }); // nothing precedes the first frame + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + expect(frameOf(await consumer.next()).timestamp).toBe(33_000 as Time.Micro); writeGroupWithLegacyFrames(track, 1, [66_000 as Time.Micro]); await settle(); - expect((await consumer.next())?.continuous).toBe(true); // group 1 continues group 0 + expect(frameOf(await consumer.next()).timestamp).toBe(66_000 as Time.Micro); // group 1 continues group 0 consumer.close(); }); -test("Consumer reports a marker group as a playhead event", async () => { +test("Consumer reads a marker group as an end, not a gap", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 2_000 as Time.Milli }); @@ -1523,13 +1505,11 @@ test("Consumer reports a marker group as a playhead event", async () => { writeGroupWithLegacyFrames(track, 2, [1_000_000 as Time.Micro]); await settle(); - expect((await consumer.next())?.discontinuity).toBe(0); - expect((await consumer.next())?.discontinuity).toBe(0); // group 0 done - const reset = await consumer.next(); - expect(reset?.frame).toBeUndefined(); - expect(reset?.discontinuity).toBe(1); - expect(reset?.continuous).toBe(false); - expect((await nextFrame(consumer))?.discontinuity).toBe(1); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + expect(await consumer.next()).toEqual({ kind: "end", group: 1, end: 0 as Time.Micro }); + // A declared stop loses nothing, so no gap follows it. + expect(frameOf(await consumer.next()).timestamp).toBe(1_000_000 as Time.Micro); consumer.close(); }); @@ -1545,52 +1525,48 @@ test("Consumer empty groups mean nothing", async () => { writeGroupWithLegacyFrames(track, 2, [1_000 as Time.Micro]); await settle(); - expect((await nextFrame(consumer))?.frame?.timestamp).toBe(0 as Time.Micro); - expect((await nextFrame(consumer))?.frame?.timestamp).toBe(1_000 as Time.Micro); - expect(consumer.discontinuity).toBe(0); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + expect(frameOf(await consumer.next()).timestamp).toBe(1_000 as Time.Micro); consumer.close(); }); -test("Consumer latency skip bumps playhead generation once", async () => { +test("Consumer reads a latency skip as one gap", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Time.Milli }); writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); await settle(); expect((await nextFrame(consumer))?.frame?.timestamp).toBe(0 as Time.Micro); - expect((await consumer.next())?.frame).toBeUndefined(); writeGroupWithLegacyFrames(track, 2, [100_000 as Time.Micro]); await settle(); - const resumed = await nextFrame(consumer); - expect(resumed?.frame?.timestamp).toBe(100_000 as Time.Micro); - expect(resumed?.discontinuity).toBe(1); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); consumer.close(); }); -test("Consumer jumps the playhead after a shed marker with a timestamp hole", async () => { +test("Consumer reads a timestamp hole as a gap", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Time.Milli }); writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); await settle(); - expect((await consumer.next())?.frame?.timestamp).toBe(0 as Time.Micro); - expect((await consumer.next())?.frame).toBeUndefined(); + expect((await nextFrame(consumer))?.frame?.timestamp).toBe(0 as Time.Micro); writeGroupWithLegacyFrames(track, 3, [1_000_000 as Time.Micro]); await settle(); - const resumed = await nextFrame(consumer); - expect(resumed?.frame?.timestamp).toBe(1_000_000 as Time.Micro); - expect(resumed?.discontinuity).toBe(1); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(1_000_000 as Time.Micro); consumer.close(); }); -test("Consumer zero-budget skip keeps a contiguous marker", async () => { +test("Consumer reads a shed marker group as a gap", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Time.Milli }); @@ -1607,16 +1583,16 @@ test("Consumer zero-budget skip keeps a contiguous marker", async () => { writeGroupWithLegacyFrames(track, 2, [500 as Time.Micro]); await settle(); - const resumed = await nextFrame(consumer); - expect(resumed?.frame?.timestamp).toBe(500 as Time.Micro); - expect(resumed?.discontinuity).toBe(1); + // The zero budget sheds the marker unread, and what it declared is lost with it. + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(500 as Time.Micro); group0.close(); consumer.close(); }); // The case the group-number heuristic got backwards: ids jump, but the PTS timeline is unbroken. -test("Consumer reports continuity across a PTS-contiguous group id jump (CMAF)", async () => { +test("Consumer yields no gap across a PTS-contiguous group id jump (CMAF)", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new CmafFormat(TEST_INIT), @@ -1628,23 +1604,19 @@ test("Consumer reports continuity across a PTS-contiguous group id jump (CMAF)", a.writeFrame(cmafFrame(0x01, 0, 0)); a.close(); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x01])); - expect((await consumer.next())?.frame).toBeUndefined(); // group-done + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); // Sequence jumps +90_000 but the first PTS meets A's end, so nothing is missing. const b = new Group.Producer(1_090_000); track.writeGroup(b); b.writeFrame(cmafFrame(0x02, 3000, 1)); - const result = await consumer.next(); - expect(result?.frame?.payload).toEqual(new Uint8Array([0x02])); - expect(result?.continuous).toBe(true); - expect(result?.discontinuity).toBe(0); + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); consumer.close(); }); -test("live duration marker follows an immediately delivered video frame", async () => { +test("a live video frame is delivered before its duration marker arrives", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("video"), @@ -1662,14 +1634,16 @@ test("live duration marker follows an immediately delivered video frame", async }); await settle(); expect(delivered).toBe(true); - expect((await read)?.frame?.timestamp).toBe(0 as Time.Micro); + expect(await read).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); group.writeFrame({ payload: encodeLegacyFrame(15_000 as Time.Micro, new Uint8Array()), timestamp: Time.Timestamp.now(), }); - const result = await consumer.next(); - expect(result?.frame).toBeUndefined(); - expect(result?.end).toBe(15_000 as Time.Micro); + group.close(); + track.close(); + // The marker is the frame's duration, not an item. + expect(await consumer.next()).toBeUndefined(); consumer.close(); track.close(); }); @@ -1700,18 +1674,18 @@ for (const code of [StreamCode.Cancel, StreamCode.Internal, StreamCode.Old, Stre await settle(); group.close(new NetError.Stream(code)); await settle(); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([1])); - expect((await consumer.next())?.frame).toBeUndefined(); + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([1])); const next = track.appendGroup(); next.writeFrame({ payload: encodeLegacyFrame(1 as Time.Micro, new Uint8Array([2])), timestamp: Time.Timestamp.now(), }); next.close(); - const resumed = await consumer.next(); - expect(resumed?.frame?.payload).toEqual(new Uint8Array([2])); - expect(resumed?.continuous).toBe(false); - expect(resumed?.frame?.keyframe).toBe(true); + // The reset cut the group's tail. + expect(await consumer.next()).toEqual({ kind: "gap" }); + const resumed = frameOf(await consumer.next()); + expect(resumed.payload).toEqual(new Uint8Array([2])); + expect(resumed.keyframe).toBe(true); expect(error).not.toHaveBeenCalled(); } finally { consumer.close(); @@ -1775,8 +1749,7 @@ for (const end of [ }); try { writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); - expect((await consumer.next())?.group).toBe(0); - expect((await consumer.next())?.frame).toBeUndefined(); + expect(await nextFrame(consumer)).toMatchObject({ group: 0 }); writeGroupWithLegacyFrames(track, 2, [1_000_000 as Time.Micro]); await settle(); const pending = consumer.next(); @@ -1784,10 +1757,8 @@ for (const end of [ const resumed = await Promise.race([pending, settle(100).then(() => "stalled" as const)]); expect(resumed).not.toBe("stalled"); if (resumed === "stalled") return; - expect(resumed?.group).toBe(2); - expect(resumed?.frame?.timestamp).toBe(1_000_000 as Time.Micro); - expect(resumed?.continuous).toBe(false); - expect((await consumer.next())?.frame).toBeUndefined(); + expect(resumed).toEqual({ kind: "gap" }); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 2, frame: { timestamp: 1_000_000 } }); if (end) await expect(consumer.next()).rejects.toBe(end); else expect(await consumer.next()).toBeUndefined(); } finally { diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index a7e563d6fd2..a0670f027e3 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -21,6 +21,23 @@ export interface ConsumerProps { maxDelay?: GetterInit; } +/** One item of a {@link Consumer.next} read, in delivery order. Mirrors Rust's `container::Read`. */ +export type Read = + /** A media frame from `group`. Only group equality means anything: numbers may skip. */ + | { kind: "frame"; group: number; frame: Frame } + /** + * Media before the next item will never reach this reader: it joined mid-track, or a group was + * skipped, evicted, truncated, or never arrived. A decoder resumes without the references or + * state that media built. + */ + | { kind: "gap" } + /** + * The publisher declared that its media stops at `end`: it paused, switched source, or finished. + * Frames that follow from the same `group` are the stopped encoder's final packets (an audio + * codec's drain), whose output at or past `end` is padding. The next group starts fresh. + */ + | { kind: "end"; group: number; end: Time.Micro }; + interface Group { consumer: Moq.Group.Consumer; frames: Frame[]; // decode order @@ -78,21 +95,18 @@ export class Consumer { // promotion guard can tell a timeline-continuous next group from one sitting after a gap. // Maintained only via #recordPresented; see its comment for the invariant. #presentedEnd?: Time.Micro; - // Group of the last frame next() returned, so it can report whether the following result - // continues that frame's timeline. Undefined until the first delivery and after a playhead event. - #deliveredGroup?: number; - // Set whenever the consumer throws content away: a group that fell `maxDelay` behind, - // a group truncated by a decode error. Reported (and - // cleared) on the first frame delivered from the next group, which is where the missing span - // sits. Only the consumer can know this, which is why next() reports it instead of leaving - // callers to guess from group numbers. - #gap = false; + // Group of the last media frame next() returned, so an endpoint after it reads as that frame's + // duration rather than a declared stop. + #mediaGroup?: number; + // Media went missing since the last item read, so the next one is preceded by a gap. Starts set: + // nothing before the first frame was read. + #lost = true; + // The item a gap was returned ahead of. + #next?: Read; // The first media timestamp of the latest delivered group, and its predecessor's start. // Endpoints and later frames can overlap the next group without moving this floor. #start?: { group: number; timestamp: Time.Micro }; #floor?: Time.Micro; - // Increments on a declared marker, an unproven delivered hole, and a latency skip. - #discontinuity = 0; // A group below the previous group start aborts the track. #error?: Error; @@ -313,22 +327,11 @@ export class Consumer { if (group.end !== undefined) this.#presentedEnd = group.end; } - // Whether delivering from group `sequence` continues the timeline of the last frame returned. - // Frames within a group are consecutive by protocol, so only a group boundary can break it, and - // there it comes down to whether anything was dropped in between. Deliberately not derived from - // group numbers: they need not be sequential, so adjacency neither proves continuity nor catches - // a group the max delay check truncated on the way past. - #continuesDelivery(sequence: number): boolean { - if (this.#deliveredGroup === undefined) return false; - return sequence === this.#deliveredGroup || !this.#gap; - } - #checkMaxDelay() { if (this.#active === undefined) return; let skipped = 0; const start = this.#groups[0]?.consumer.sequence; - let hole = false; // Keep skipping the oldest group while the buffered span exceeds the max delay. // This also handles gaps in group sequence numbers: if #active points to a missing @@ -359,17 +362,16 @@ export class Consumer { const nextStart = this.#groups[0]?.frames.at(0)?.timestamp ?? this.#groups[0]?.end; const marker = !first.empty && !first.media; - if (marker || !ptsContiguous(first.end ?? this.#presentedEnd, nextStart)) { - hole = true; + // Buffered frames are thrown away, and `first.end` counts them, so contiguity alone would hide them. + const discarded = first.frames.length > 0; + if (marker || discarded || !ptsContiguous(first.end ?? this.#presentedEnd, nextStart)) { + this.#lost = true; } first.consumer.close(); first.frames.length = 0; skipped++; - this.#gap = true; } - if (hole) this.#markPlayhead(); - if (skipped) { console.warn( `skipping slow groups: track=${this.#track.name} ${start} -> ${this.#active} count=${skipped}`, @@ -441,35 +443,17 @@ export class Consumer { } /** - * Returns the next frame in order along with its group number and the current - * {@link discontinuity} count, awaiting one if needed. A `frame` of undefined signals either - * the end of that group or, when `end` is present, an exclusive media endpoint carried by a - * legacy marker. The overall result is undefined once closed. When `discontinuity` - * jumps relative to the previous call, re-apply startup delay and skip: it is a playhead - * event, and the next frame may be a delta that continues the interrupted group. - * - * `continuous` is true when this result picks up exactly where the previous frame left off, so - * the span between them can be treated as delivered. It is false on the first frame, after a - * playhead event, and whenever the consumer threw content away to keep up: a slow group skipped - * for the max delay, a group truncated by a decode error. Use it rather than comparing group - * numbers, which are not required to be sequential: adjacency neither proves the timeline is - * unbroken nor catches a group dropped on the way past. - * - * It reports what this consumer dropped plus marker groups the publisher declared. - * An unmarked forward timestamp jump still reads as continuous because nothing on the wire says - * the missing span will never arrive. + * Returns the next {@link Read} in delivery order, awaiting one if needed: a frame, a gap where + * media will never arrive (always one before the first frame), or an end the publisher declared. + * An unmarked forward timestamp jump is not a gap, since nothing on the wire says media is missing. * After buffered groups drain, a finished track returns undefined and an aborted track throws. */ - async next(): Promise< - | { - frame: Frame | undefined; - group: number; - discontinuity: number; - continuous: boolean; - end?: Time.Micro; - } - | undefined - > { + async next(): Promise { + if (this.#next) { + const next = this.#next; + this.#next = undefined; + return next; + } for (;;) { if (this.#error) throw this.#error; this.#checkMalformed(); @@ -487,8 +471,8 @@ export class Consumer { // fallback fired because no later group was buffered yet, and the real (large-gap, // non-sequential) next group has since arrived -- promote #active to the first real // group so delivery resumes instead of stalling on a nonexistent sequence. - // Before any media is delivered there is no timeline hole to wait for. #start survives - // playhead events, unlike #deliveredGroup, so this exception only applies at startup. + // Before any media is delivered there is no timeline hole to wait for, so this exception + // only applies at startup. // Otherwise promote #active when the first buffered group continues the timeline we left off at, // when a completed empty group can be walked (empty groups mean nothing), or when the hole // is proven: the head already reaches past where presentation left off by more than the @@ -511,9 +495,7 @@ export class Consumer { head.latest !== undefined && head.latest - this.#presentedEnd > maxDelay)); if (startup || empty || contiguous || skipHole || ended !== undefined) { - if (!startup && (skipHole || ended !== undefined) && !contiguous && !empty) - this.#markPlayhead(); - if (!contiguous) this.#gap = true; + if (!contiguous && !empty) this.#lost = true; this.#active = head.consumer.sequence; } } @@ -529,19 +511,12 @@ export class Consumer { const seq = this.#groups[0].consumer.sequence; const end = this.#format.end?.(frame); if (end !== undefined) { - if (!this.#groups[0].media) this.#markPlayhead(); this.#updateBuffered(); - return { - frame: undefined, - group: seq, - discontinuity: this.#discontinuity, - continuous: this.#continuesDelivery(seq), - end, - }; + // A trailing endpoint is the last frame's duration, which only the skip logic needs. + if (seq === this.#mediaGroup) continue; + return this.#emit({ kind: "end", group: seq, end }); } - const continuous = this.#continuesDelivery(seq); - if (seq !== this.#deliveredGroup) this.#gap = false; - this.#deliveredGroup = seq; + this.#mediaGroup = seq; if (this.#start === undefined || seq > this.#start.group) { this.#floor = this.#start?.timestamp; @@ -553,7 +528,7 @@ export class Consumer { throw this.#error; } this.#updateBuffered(); - return { frame, group: seq, discontinuity: this.#discontinuity, continuous }; + return this.#emit({ kind: "frame", group: seq, frame }); } // Check if the group is done and then remove it. A group is removable only @@ -575,17 +550,9 @@ export class Consumer { const group = this.#groups.shift(); if (group) { - const seq = group.consumer.sequence; - if (group.truncated) this.#gap = true; + if (group.truncated) this.#lost = true; this.#updateBuffered(); - return { - frame: undefined, - group: seq, - discontinuity: this.#discontinuity, - // A marker carries no content of its own, so this just reports whether - // the group it closes was itself reached without a gap. - continuous: this.#continuesDelivery(seq), - }; + continue; } } @@ -616,10 +583,12 @@ export class Consumer { } } - #markPlayhead(): void { - this.#discontinuity++; - this.#deliveredGroup = undefined; - this.#gap = true; + // Return `read`, or a gap ahead of it when media went missing first. + #emit(read: Read): Read { + if (!this.#lost) return read; + this.#lost = false; + this.#next = read; + return { kind: "gap" }; } #updateBuffered(): void { @@ -648,14 +617,6 @@ export class Consumer { this.#buffered.set(ranges); } - /** - * A counter that increments at each playhead event: a declared marker group, an unproven - * delivered hole, or a latency skip. Also surfaced per-read via {@link next}. - */ - get discontinuity(): number { - return this.#discontinuity; - } - /** Stop consuming and release the track and all buffered groups. */ close(): void { this.#finish(null); diff --git a/js/hang/src/container/index.ts b/js/hang/src/container/index.ts index 5ef19f7f7a8..135c5600d8f 100644 --- a/js/hang/src/container/index.ts +++ b/js/hang/src/container/index.ts @@ -7,7 +7,7 @@ export * as Loc from "@moq/loc"; export * as Cmaf from "./cmaf"; -export { Consumer, type ConsumerProps } from "./consumer"; +export { Consumer, type ConsumerProps, type Read } from "./consumer"; export type { Format } from "./format"; export * as Legacy from "./legacy"; export { type TrackInfoOptions, trackInfo } from "./track"; diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 77da93f5c88..15c8b468035 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -67,15 +67,14 @@ class FakeData { close() {} } -type Read = NonNullable>>; +type Read = Container.Read; let frameTimestamp = 0; -function frame(): Read { +function frame(): Extract { frameTimestamp += 20_000; return { + kind: "frame", group: 0, - discontinuity: 0, - continuous: true, frame: { timestamp: Time.Micro(frameTimestamp), payload: new Uint8Array([1]), keyframe: true }, }; } @@ -234,7 +233,7 @@ async function play(initial: Delay, buffered = false, cmaf = false) { const timestamps: number[] = []; for (let i = 0; i < count; i++) { const read = frame(); - timestamps.push(read.frame?.timestamp ?? 0); + timestamps.push(read.frame.timestamp); push(read); await microtasks(); } @@ -291,13 +290,12 @@ async function play(initial: Delay, buffered = false, cmaf = false) { }; } -it("decodes an audio packet inside a group after a playhead discontinuity", async () => { +it("decodes an audio packet inside a group after a gap", async () => { const playback = await play(Time.Milli(100)); try { await playback.play(); + playback.push({ kind: "gap" }); const next = frame(); - next.discontinuity = 1; - if (!next.frame) throw new Error("missing test frame"); next.frame.keyframe = false; playback.push(next); await microtasks(); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 348f378ff96..d618f87db53 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -119,7 +119,7 @@ export class Decoder { // The rendition is absent and the ring has drained its tail, so the graph has nothing to play. readonly #idle: Computed; - // Ordered discontinuity and endpoint state from the container consumer. + // Ordered epoch and endpoint state from the container consumer. #terminal = new Terminal(); // The container consumer's arrival estimate, unset while nothing is subscribed. @@ -471,16 +471,12 @@ export class Decoder { for (;;) { const next = await nextMedia(consumer); if (!next) break; - if (this.#onNext(next)) { + if (this.#onRead(next)) { decoder.reset(); decoder.configure(decoderConfig); } - if (next.end !== undefined) { - continue; - } - + if (next.kind !== "frame") continue; const { frame } = next; - if (!frame) continue; // Mark that we received this frame right now. const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro); @@ -581,13 +577,12 @@ export class Decoder { if (!next) break; // Reset and re-anchor before decoding the first frame of a new codec epoch. - if (this.#onNext(next)) { + if (this.#onRead(next)) { decoder.reset(); decoder.configure(decoderConfig); } - + if (next.kind !== "frame") continue; const { frame } = next; - if (!frame) continue; const timestamp = Time.Milli.fromMicro(frame.timestamp); this.sync.received(timestamp, "audio"); @@ -732,15 +727,10 @@ export class Decoder { this.#ring.peek()?.reset(); } - // Apply ordered container metadata before handling the result. An endpoint that also - // starts a new epoch must survive the reset so its following drain is trimmed. - #onNext(next: { - discontinuity: number; - group: number; - end?: Time.Micro; - frame?: { timestamp: Time.Micro }; - }): boolean { - if (!this.#terminal.update(next)) return false; + // Apply an ordered container read before handling it, reporting whether a codec epoch ended: + // a gap, or a later group after a declared end. + #onRead(read: Container.Read): boolean { + if (!this.#terminal.update(read)) return false; this.#ring.peek()?.reset(); this.sync.reset(); return true; diff --git a/js/watch/src/audio/playout.test.ts b/js/watch/src/audio/playout.test.ts index 362681f2186..e2e05d9e944 100644 --- a/js/watch/src/audio/playout.test.ts +++ b/js/watch/src/audio/playout.test.ts @@ -89,7 +89,7 @@ async function measure(t: Arrival[], budget: (target: Time.Milli) => Time.Milli) for (;;) { const next = await consumer.next(); if (!next) return; - if (next.frame) result.delivered++; + if (next.kind === "frame") result.delivered++; } })(); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index 3c18faf97ba..e7ae9424f08 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -3,7 +3,7 @@ * * Every frame is written into its group at the instant the trace says it arrived, and the player's * real `Container.Consumer` decides what to deliver, wait for, and skip, at the max delay the delay - * sets. Delivered frames go into the real ring, and a discontinuity resets it, the way the decoder + * sets. Delivered frames go into the real ring, and a new codec epoch resets it, the way the decoder * does. One render quantum is read every quantum's worth of that same clock, which is what the * AudioWorklet does. A real {@link Sync} resolves the delay from the playout target the decoder * registers, measured by that same consumer from the recorded catalog config, and the ring follows @@ -221,7 +221,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const next = await consumer.next(); if (!next) return; if (terminal.update(next)) ring.reset(); - if (next.end !== undefined || !next.frame) continue; + if (next.kind !== "frame") continue; const { timestamp } = next.frame; const length = samples.get(timestamp); diff --git a/js/watch/src/audio/terminal.test.ts b/js/watch/src/audio/terminal.test.ts index 6c6d9607cad..44eb0d06c33 100644 --- a/js/watch/src/audio/terminal.test.ts +++ b/js/watch/src/audio/terminal.test.ts @@ -2,21 +2,31 @@ import { expect, test } from "bun:test"; import type { Time } from "@moq/net"; import { Terminal } from "./terminal"; +const frame = (group: number, timestamp: number) => ({ + kind: "frame" as const, + group, + frame: { timestamp: timestamp as Time.Micro }, +}); +const end = (group: number, timestamp: number) => ({ kind: "end" as const, group, end: timestamp as Time.Micro }); +const gap = { kind: "gap" as const }; + test("Terminal trims a partial packet to the source endpoint", () => { const terminal = new Terminal(); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro }, end: 17_916 as Time.Micro }); + terminal.update(end(0, 17_916)); + terminal.update(frame(0, 0)); expect(terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(860); }); test("Terminal maps Opus delay back before trimming terminal output", () => { const terminal = new Terminal(); terminal.clear(312); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro } }); + terminal.update(frame(0, 0)); const body = terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }); expect(body).toEqual({ timestamp: 0 as Time.Micro, frameOffset: 312, frames: 648 }); - terminal.update({ discontinuity: 0, group: 0, end: 20_000 as Time.Micro }); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 20_000 as Time.Micro } }); + expect(terminal.update(end(1, 20_000))).toBe(false); + // The drain shares the end's group, so it continues the epoch. + expect(terminal.update(frame(1, 20_000))).toBe(false); const drain = terminal.span({ timestamp: 20_000, sampleRate: 48_000, numberOfFrames: 960 }); expect(drain).toEqual({ timestamp: 13_500 as Time.Micro, frameOffset: 0, frames: 312 }); expect(body.frames + drain.frames).toBe(960); @@ -24,41 +34,48 @@ test("Terminal maps Opus delay back before trimming terminal output", () => { test("Terminal preserves samples before the source endpoint", () => { const terminal = new Terminal(); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro }, end: 20_000 as Time.Micro }); + terminal.update(end(0, 20_000)); + terminal.update(frame(0, 0)); expect(terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(960); }); -test("a rewound endpoint survives the discontinuity reset and trims its terminal packet", () => { +test("the gap before the first frame ends no epoch", () => { + const terminal = new Terminal(); + expect(terminal.update(gap)).toBe(false); +}); + +test("an endpoint after a gap trims its own group", () => { const terminal = new Terminal(); terminal.clear(312); - terminal.update({ discontinuity: 0, group: 0, end: 40_000 as Time.Micro }); + terminal.update(frame(0, 0)); + expect(terminal.update(gap)).toBe(true); - const reset = terminal.update({ discontinuity: 1, group: 1, end: 20_000 as Time.Micro }); - expect(reset).toBe(true); + terminal.update(end(1, 20_000)); expect(terminal.end).toBe(20_000 as Time.Micro); - - terminal.update({ discontinuity: 1, group: 1, frame: { timestamp: 20_000 as Time.Micro } }); + terminal.update(frame(1, 20_000)); const span = terminal.span({ timestamp: 20_000, sampleRate: 48_000, numberOfFrames: 960 }); expect(span).toEqual({ timestamp: 20_000 as Time.Micro, frameOffset: 312, frames: 0 }); }); -test("a discontinuity reapplies Opus pre-skip in the resumed epoch", () => { +test("a gap reapplies Opus pre-skip in the resumed epoch", () => { const terminal = new Terminal(); terminal.clear(312); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro } }); + terminal.update(frame(0, 0)); expect(terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }).frameOffset).toBe(312); - terminal.update({ discontinuity: 1, group: 1, frame: { timestamp: 1_000_000 as Time.Micro } }); + expect(terminal.update(gap)).toBe(true); + terminal.update(frame(1, 1_000_000)); const resumed = terminal.span({ timestamp: 1_000_000, sampleRate: 48_000, numberOfFrames: 960 }); expect(resumed).toEqual({ timestamp: 1_000_000 as Time.Micro, frameOffset: 312, frames: 648 }); }); test("an endpoint does not trim media resumed in a later group", () => { const terminal = new Terminal(); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro } }); + terminal.update(frame(0, 0)); // A discontinuity marker: its own group, ending the epoch at the live edge. - terminal.update({ discontinuity: 1, group: 1, end: 20_000 as Time.Micro }); - terminal.update({ discontinuity: 1, group: 2, frame: { timestamp: 5_000_000 as Time.Micro } }); + expect(terminal.update(end(1, 20_000))).toBe(false); + // The next group comes from a fresh encoder. + expect(terminal.update(frame(2, 5_000_000))).toBe(true); expect(terminal.end).toBeUndefined(); expect(terminal.span({ timestamp: 5_000_000, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(960); }); diff --git a/js/watch/src/audio/terminal.ts b/js/watch/src/audio/terminal.ts index f89a8b15e74..59e501ae176 100644 --- a/js/watch/src/audio/terminal.ts +++ b/js/watch/src/audio/terminal.ts @@ -6,12 +6,11 @@ interface SampleSpan { readonly numberOfFrames: number; } -interface Update { - readonly discontinuity: number; - readonly group: number; - readonly end?: Time.Micro; - readonly frame?: { readonly timestamp: Time.Micro }; -} +/** The parts of a container read the terminal tracks. */ +type Update = + | { readonly kind: "frame"; readonly group: number; readonly frame: { readonly timestamp: Time.Micro } } + | { readonly kind: "gap" } + | { readonly kind: "end"; readonly group: number; readonly end: Time.Micro }; /** The portion of decoded audio that belongs to the source timeline. */ export interface DecodedSpan { @@ -25,7 +24,8 @@ export interface DecodedSpan { /** Track codec epochs and map decoded output onto the source timeline. */ export class Terminal { - #discontinuity = 0; + // Whether a frame arrived since the last reset, so there is an epoch to end. + #started = false; #end?: Time.Micro; // The group that carried `#end`. It bounds only the terminal packets that follow it there. #endGroup?: number; @@ -40,30 +40,39 @@ export class Terminal { /** Reset state for a new container consumer with Opus pre-skip in 48 kHz frames. */ clear(preSkip = 0): void { - this.#discontinuity = 0; + this.#started = false; this.#end = undefined; + this.#endGroup = undefined; this.#preSkip = preSkip; this.#resetEpoch(); } - /** Apply one ordered consumer result and report whether its codec epoch changed. */ - update(next: Update): boolean { - const reset = next.discontinuity !== this.#discontinuity; - if (reset) { - this.#discontinuity = next.discontinuity; - this.#end = undefined; - this.#resetEpoch(); - } - // A frame from a later group is new media, e.g. resuming after a discontinuity marker. - if (next.frame && next.group !== this.#endGroup) this.#end = undefined; - if (next.frame && this.#epoch === undefined) this.#epoch = next.frame.timestamp; - if (next.end !== undefined) { - this.#end = next.end; - this.#endGroup = next.group; + /** Apply one ordered consumer read and report whether a codec epoch ended before it. */ + update(read: Update): boolean { + if (read.kind === "gap") return this.#restart(); + if (read.kind === "end") { + this.#end = read.end; + this.#endGroup = read.group; + return false; } + // A later group after a declared end comes from a fresh encoder; the end's own group carries + // the stopped encoder's final packets. + const reset = this.#end !== undefined && read.group !== this.#endGroup && this.#restart(); + this.#started = true; + this.#epoch ??= read.frame.timestamp; return reset; } + // Start a new epoch, reporting whether one was running. + #restart(): boolean { + const started = this.#started; + this.#started = false; + this.#end = undefined; + this.#endGroup = undefined; + this.#resetEpoch(); + return started; + } + /** Remove codec pre-skip and terminal padding from one decoded sample. */ span(sample: SampleSpan): DecodedSpan { const epoch = this.#epoch ?? (sample.timestamp as Time.Micro); diff --git a/js/watch/src/retention.test.ts b/js/watch/src/retention.test.ts index 453c545f137..457b2ec975d 100644 --- a/js/watch/src/retention.test.ts +++ b/js/watch/src/retention.test.ts @@ -100,10 +100,10 @@ test("a long subscription through the player path leaves nothing behind", async const play = async (count: number) => { for (let i = 0; i < count; i++) { producer.encode(new Uint8Array([i & 0xff]), Math.round(Time.Micro.now()) as Time.Micro, true); - // A new group closes the previous one with a duration marker, which reads as no frame. + // Pass over the gap before the first frame. let next = await nextMedia(consumer); - while (next && !next.frame) next = await nextMedia(consumer); - if (!next?.frame) throw new Error("the track ended"); + while (next && next.kind !== "frame") next = await nextMedia(consumer); + if (!next) throw new Error("the track ended"); const timestamp = Time.Milli.fromMicro(next.frame.timestamp); sync.received(timestamp); diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index d18bc4775be..e2acb831415 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -173,15 +173,15 @@ class Picture { } } -type Read = NonNullable>>; -function sample(group: number, timestamp: number, discontinuity = 0, keyframe = true): Read { +type Read = Container.Read; +function sample(group: number, timestamp: number, keyframe = true): Read { return { + kind: "frame", group, - discontinuity, - continuous: true, frame: { timestamp: Time.Micro(timestamp), payload: new Uint8Array([1]), keyframe }, }; } +const gap: Read = { kind: "gap" }; async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: readonly number[] = []) { const originalDecoder = Object.getOwnPropertyDescriptor(globalThis, "VideoDecoder"); @@ -420,20 +420,20 @@ it("clears the picture once no rendition is selectable", async () => { } }); -// A late marker from older history raises a discontinuity without making older groups newer. -function marker(group: number, end: number, discontinuity: number): Read { - return { group, discontinuity, continuous: false, frame: undefined, end: Time.Micro(end) }; +// A late marker from older history breaks the timeline without making older groups newer. +function marker(group: number, end: number): Read { + return { kind: "end", group, end: Time.Micro(end) }; } for (const kind of ["legacy", "cmaf"] as const) { - it(`${kind} skips older groups before decode, even across a stale marker's discontinuity`, async () => { + it(`${kind} skips older groups before decode, even across a stale marker`, async () => { const playback = await guardedPlayback(kind, [ sample(10, 1000), sample(9, 900), sample(10, 1100), - marker(8, 850, 1), - sample(7, 700, 1), - sample(11, 1200, 1), + marker(8, 850), + sample(7, 700), + sample(11, 1200), ]); try { expect(playback.submitted).toEqual([1000, 1100, 1200]); @@ -446,9 +446,9 @@ for (const kind of ["legacy", "cmaf"] as const) { // A group may restart at its own start, so a picture already queued from later in that // group is above the new keyframe. It must not come out afterwards and late-reject it. - it(`${kind} drops a picture queued before a discontinuity`, async () => { + it(`${kind} drops a picture queued before a gap`, async () => { const queued = 60_000_000; - const playback = await guardedPlayback(kind, [sample(1, 0), sample(1, queued), sample(2, 0, 1)], [queued]); + const playback = await guardedPlayback(kind, [sample(1, 0), sample(1, queued), gap, sample(2, 0)], [queued]); try { expect(playback.submitted).toEqual([0, queued, 0]); expect(playback.outputs).toEqual([0, 0]); @@ -459,15 +459,15 @@ for (const kind of ["legacy", "cmaf"] as const) { } }); - // A stale marker (or a max-delay hole) can bump the playhead while the live group is only - // partly delivered, so the next frame is a delta that still needs the decoder's references. - it(`${kind} keeps decoding a delta that follows a discontinuity`, async () => { + // A stale marker can break the timeline while the live group is only partly delivered, so the + // next frame is a delta that still needs the decoder's references. + it(`${kind} keeps decoding a delta that follows a stale marker`, async () => { const playback = await guardedPlayback(kind, [ sample(10, 1000), - sample(10, 1100, 0, false), - marker(8, 850, 1), - sample(10, 1200, 1, false), - sample(11, 1300, 1), + sample(10, 1100, false), + marker(8, 850), + sample(10, 1200, false), + sample(11, 1300), ]); try { expect(playback.submitted).toEqual([1000, 1100, 1200, 1300]); @@ -603,7 +603,7 @@ it("a shared audio clock reset cannot present a parked video frame", async () => playback.enabled.set(false); playback.delay.set(Time.Milli(1000)); await microtasks(); - reads.push(sample(0, 0), sample(0, 100_000, 0, false)); + reads.push(sample(0, 0), sample(0, 100_000, false)); playback.enabled.set(true); await microtasks(); expect(playback.submitted).toEqual([0, 100_000]); @@ -645,7 +645,7 @@ it("closing the track releases a video wait parked across a reset", async () => playback.enabled.set(false); playback.delay.set(Time.Milli(1000)); await microtasks(); - reads.push(sample(0, 0), sample(0, 100_000, 0, false)); + reads.push(sample(0, 0), sample(0, 100_000, false)); playback.enabled.set(true); await microtasks(); expect(playback.submitted).toEqual([0, 100_000]); @@ -680,7 +680,7 @@ it("a reset drops a parked video frame even when the next anchor restores the re playback.enabled.set(false); playback.delay.set(Time.Milli(1000)); await microtasks(); - reads.push(sample(0, 0), sample(0, 100_000, 0, false)); + reads.push(sample(0, 0), sample(0, 100_000, false)); playback.enabled.set(true); await microtasks(); expect(playback.submitted).toEqual([0, 100_000]); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index a1f915fee1f..173047d44f0 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -410,9 +410,6 @@ class DecoderTrack { // Decoded frames waiting to be rendered. #buffered = new Signal([]); - // The last discontinuity count seen from the container consumer. - #discontinuity = 0; - // Codec description from the last configure, reused after a playhead reset. #description?: Uint8Array; @@ -543,15 +540,18 @@ class DecoderTrack { const next = await nextMedia(consumer); if (!next) break; - // Playhead moved: re-anchor before decoding further. - if (this.#onDiscontinuity(next.discontinuity)) previous = undefined; - + // A gap or a declared end breaks the timeline: re-anchor before decoding further, and + // never bridge the buffer across it. + if (next.kind !== "frame") { + if (latest !== undefined) this.#onBreak(); + previous = undefined; + continue; + } const { frame } = next; - if (!frame) continue; // The group is done // An older keyframe would replace the codec references needed by live deltas. Group - // sequences only grow within a subscription, so a discontinuity (which a stale marker - // can raise) never resets this guard. Rejected payloads still count as received bytes. + // sequences only grow within a subscription, so a break (which a stale marker can + // raise) never resets this guard. Rejected payloads still count as received bytes. const stale = latest !== undefined && next.group < latest; this.stats.update((current) => ({ frameCount: (current?.frameCount ?? 0) + (stale ? 0 : 1), @@ -570,11 +570,11 @@ class DecoderTrack { }); // Track decode buffer: frames sent to decoder but not yet rendered. Only bridge from - // the previous frame when the consumer says nothing is missing in between. Group ids - // can't answer that: they aren't required to be sequential (some encoders derive them - // from DTS), so adjacency neither proves continuity nor rules out a gap the consumer - // skipped, and reporting a skipped span as decoded overstates the buffer. - if (previous !== undefined && next.continuous) { + // the previous frame when no gap or end came between them. Group ids can't answer that: + // they aren't required to be sequential (some encoders derive them from DTS), so + // adjacency neither proves continuity nor rules out a gap the consumer skipped, and + // reporting a skipped span as decoded overstates the buffer. + if (previous !== undefined) { this.#addBuffered(Time.Milli.fromMicro(previous), Time.Milli.fromMicro(frame.timestamp)); } @@ -623,15 +623,17 @@ class DecoderTrack { const next = await nextMedia(consumer); if (!next) break; - // Playhead moved: re-anchor before decoding further. - if (this.#onDiscontinuity(next.discontinuity)) previous = undefined; - + // A gap or a declared end breaks the timeline (see #runLegacy). + if (next.kind !== "frame") { + if (latest !== undefined) this.#onBreak(); + previous = undefined; + continue; + } const { frame } = next; - if (!frame) continue; // An older keyframe would replace the codec references needed by live deltas. Group - // sequences only grow within a subscription, so a discontinuity (which a stale marker - // can raise) never resets this guard. Rejected payloads still count as received bytes. + // sequences only grow within a subscription, so a break (which a stale marker can + // raise) never resets this guard. Rejected payloads still count as received bytes. const stale = latest !== undefined && next.group < latest; this.stats.update((current) => ({ frameCount: (current?.frameCount ?? 0) + (stale ? 0 : 1), @@ -643,9 +645,9 @@ class DecoderTrack { const timestamp = Time.Milli.fromMicro(frame.timestamp); this.sync.received(timestamp, "video"); - // Track decode buffer (see #runLegacy: bridge on the consumer's continuity signal, + // Track decode buffer (see #runLegacy: bridge only when no gap or end came between, // never on group adjacency, which proves nothing about the timeline). - if (previous !== undefined && next.continuous) { + if (previous !== undefined) { this.#addBuffered(Time.Milli.fromMicro(previous), Time.Milli.fromMicro(frame.timestamp)); } @@ -665,25 +667,21 @@ class DecoderTrack { }); } - // The container bumped its playhead (a marker, a hole, or a latency skip). Re-anchor and - // let the next decoded frame decide the decoder reset (see #resetAtKeyframe). Clearing - // `timestamp` keeps a stale high value from late-rejecting the following frames, which may - // overlap the previous group's tail. The held picture stays until the new keyframe renders. - // Returns true when a new generation was handled. - #onDiscontinuity(count: number): boolean { - if (count === this.#discontinuity) return false; - this.#discontinuity = count; + // The container read a gap or a declared end after media. Re-anchor and let the next decoded + // frame decide the decoder reset (see #resetAtKeyframe). Clearing `timestamp` keeps a stale + // high value from late-rejecting the following frames, which may overlap the previous group's + // tail. The held picture stays until the new keyframe renders. + #onBreak(): void { this.timestamp.set(undefined); this.#buffered.set([]); this.sync.reset(); this.#resetPending = true; - return true; } - // Call before each decode. After a playhead bump, a keyframe starts a new timeline, so drop - // the pictures still queued in the decoder: one from later in a group that restarts would + // Call before each decode. After a break, a keyframe starts a new timeline, so drop the + // pictures still queued in the decoder: one from later in a group that restarts would // otherwise come out above the new keyframe and late-reject it. A delta continues a group the - // bump interrupted (a stale marker or a max-delay hole), so it keeps the decoder's references; + // break interrupted (a stale marker or a max-delay hole), so it keeps the decoder's references; // a reset would make WebCodecs require a key chunk and throw. `reset()` cannot cancel an // output callback that already holds a frame; the clock epoch it captured rejects that one. #resetAtKeyframe(decoder: VideoDecoder, keyframe: boolean): void { diff --git a/quest/m1/README.md b/quest/m1/README.md index ad8ca3fbe73..b65c148aeef 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -167,7 +167,6 @@ folded into Dropped sources. - [Encoder colour](/quest/m1/color-model.md) - every moq-video encode path signals the colour its output actually has, or refuses instead of mislabelling - [T-STD TS export](/quest/m1/tstd/README.md) - `moq export ts` is a proper remux that passes the T-STD buffer model, starting with a fixed `--delay` - [ATSC AC-3](/quest/m1/ts-atsc-ac3.md) - `export ts` carries 48 kHz ATSC AC-3 at every A/52 rate up to 640 kb/s instead of aborting from 384 kb/s up -- [Rust video delivery continuity](/quest/m1/video-continuous.md) - Rust video decode reads report each picture's delivery continuity, like the container and audio reads, for the tune-in and warmup trims - [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - a viewer joining at a recovery point drops the leading pictures it cannot decode; continuous viewers keep them - [Audio warmup](/quest/m1/audio-warmup.md) - a viewer joining an Opus rendition mid-stream never hears the unconverged first 80 ms - [#3021](/quest/m1/3021-moq-gst-anchor-generated-media-timelines-to-wall-clock.md) - moq-gst picks the broadcast wall epoch; a restarted source is a new epoch, not a forward re-anchor diff --git a/quest/m1/audio-warmup.md b/quest/m1/audio-warmup.md index 02c92f3a1ac..1f8888b360a 100644 --- a/quest/m1/audio-warmup.md +++ b/quest/m1/audio-warmup.md @@ -14,9 +14,9 @@ frames decode independently and set nothing; HE-AAC is out of scope. - `rs/moq-audio/src/encode/producer.rs` and `rs/moq-mux/src/codec/opus` publish `warmup` for Opus; `js/publish` does the same for its Opus track. - `rs/moq-audio/src/decode` already trims Opus `pre_skip` at stream start; - the warmup trim is the same mechanism keyed on the container consumer's - non-continuous signal (`continuous` on the Rust audio decoder's - `container::Delivery`), and the + the warmup trim is the same mechanism keyed on the container read's gap + item (`container::Read::Gap` in Rust, `{ kind: "gap" }` in JS), never on a + declared end, which would cut 80 ms at every pause, and the subscription's maximum age grows by `warmup` as the video consumer quest does. `js/watch` audio mirrors it. - The trim replaces `LEGACY_WARMUP_CALLBACKS` in `js/watch/src/audio/decoder.ts`, diff --git a/quest/m1/open-gop-leading-pictures.md b/quest/m1/open-gop-leading-pictures.md index 4ad437f5261..1fa124841f3 100644 --- a/quest/m1/open-gop-leading-pictures.md +++ b/quest/m1/open-gop-leading-pictures.md @@ -39,19 +39,16 @@ decode, and the JS test should drive the software decoder. - In the JS consumer (`js/hang/src/container/consumer.ts` forces the first sample of a group to `keyframe`, and `js/watch/src/video/decoder.ts` submits - it as `"key"`): for the first group after any non-continuous transition, - skip delta frames stamped before that group's keyframe. That covers a - subscribe, a declared discontinuity, and a latency skip: `#checkMaxDelay` - records the skip through `#gap` and `next()` reports the next frame with - `continuous: false`. Latency skip also bumps playhead generation (startup - delay) but does not flush the decoder. Leading pictures after that - non-continuous transition are still skipped, as above; a viewer that skipped - into a later GOP lacks its references just like a cold join. - Every continuous group is passed through untouched. + it as `"key"`): for the first group after a gap item, skip delta frames + stamped before that group's keyframe. `next()` yields `{ kind: "gap" }` + before the first frame of a subscribe and after a latency skip, eviction, or + hole; a declared end is not a gap, since a fresh encoder has no earlier GOP + to reference. A latency skip does not flush the decoder, and a viewer that + skipped into a later GOP lacks its references just like a cold join. + Every group without a gap before it is passed through untouched. - The same rule in the Rust decode path (`moq-video` decode consumers), so native playback and the transcoder tune in the same way, keyed on the - non-continuous signal from [Rust video delivery - continuity](/quest/m1/video-continuous.md). + container read's `Read::Gap` item. - The same rule in `moq export ts`. Decided (2026-10-01): the fixed-delay export still sends a join's orphaned leading pictures, so 3 of 500 frames on the open-GOP fixture decode after they present @@ -70,12 +67,8 @@ decode, and the JS test should drive the software decoder. group's leading pictures trimmed too, so an implementation that only trims the initial group fails. Both cases in both languages. -Decided 2026-10-08: the Rust non-continuous signal is its own quest, so -audio warmup and consumer warmup don't wait on this one. - -## Required - -- [Rust video delivery continuity](/quest/m1/video-continuous.md) - the Rust decode path keys the trim on it +Decided 2026-10-12: both container reads yield the gap item, so nothing +here waits on another quest. ## Related diff --git a/quest/m1/video-continuous.md b/quest/m1/video-continuous.md deleted file mode 100644 index 2c3b47db74d..00000000000 --- a/quest/m1/video-continuous.md +++ /dev/null @@ -1,39 +0,0 @@ -# [M] Rust video delivery continuity - -## Goal - -`moq_video::decode::Consumer::read` returns `container::Delivery`, so -each decoded picture says whether it continues the previous delivery, like -the container and audio reads already do. A picture decoded from input before -a marker, skip, or subscribe keeps that input's flag even when the codec -returns it late, and the first picture after the boundary reads -non-continuous. - -## Plan - -Split from the container and audio work in -[#5288](https://github.com/moq-dev/moq/pull/5288), where the maintainer chose -the `container::Delivery { frame, continuous }` wrapper and landing video -separately. - -A decoder can return a picture from before a marker while processing the -access unit after it, or only on flush, so a single pending boolean attaches -the boundary to the wrong picture. Carry a private input identity through -every backend (`rs/moq-video/src/decode/backend`) and map each output back to -its input's flag. A map keyed only by presentation time is not enough: equal -timestamps can straddle a marker. VideoToolbox and Media Foundation already -map synthetic sample times, OpenH264 queues presentation times, and VAAPI, -NVDEC and MediaCodec carry codec timestamps. An output whose input identity -did not survive (Media Foundation's unmatched-output fallback) is an explicit -error, never a borrowed flag. Keep dropped-input bookkeeping bounded. - -Tests: delayed output on both sides of a marker, duplicate timestamps, and -decoder recovery, on each backend CI can run. - -Public API: breaking in moq-video (the read result). Bindings keep their -existing frame records. Wire: none. - -## Related - -- [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - the Rust decode trim keys on this signal -- [Consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md) - withholds frames after the same signal diff --git a/quest/m2/intra-refresh/consumer-warmup.md b/quest/m2/intra-refresh/consumer-warmup.md index 350a6951ae3..e7c59349ef2 100644 --- a/quest/m2/intra-refresh/consumer-warmup.md +++ b/quest/m2/intra-refresh/consumer-warmup.md @@ -19,14 +19,12 @@ This absorbs the old gradual-recovery quest, which planned to carry `recovery_frame_cnt` on the wire and count `frame_num`. The catalog duration replaces both: the rule is timestamp arithmetic on the group start. -- Withhold rule, keyed on the same non-continuous signal in both consumers: - `js/hang/src/container/consumer.ts` `next()` reports `continuous: false` - after a subscribe, a declared discontinuity, or any skip (`#gap`). The Rust - `moq_mux::container::Consumer` reports the equivalent as - `container::Delivery::continuous`; this quest reuses it once - `rs/moq-video/src/decode/consumer.rs` propagates it in - [Rust video delivery continuity](/quest/m1/video-continuous.md). For the first - group after that signal, every frame is decoded (the decoder needs them to +- Withhold rule, keyed on the gap item both container reads yield before the + first frame and after any skip: `{ kind: "gap" }` from + `js/hang/src/container/consumer.ts` `next()` and `Read::Gap` from the Rust + `moq_mux::container::Consumer`, which `rs/moq-video/src/decode/consumer.rs` + reads directly. A declared end is not a gap and keeps everything. For the + first group after a gap, every frame is decoded (the decoder needs them to build reference state) and frames stamped below `group.start + warmup` are not presented; frames stamped at or above that boundary are, so the recovery picture itself is never withheld. `js/watch/src/video/decoder.ts` and @@ -57,10 +55,6 @@ replaces both: the rule is timestamp arithmetic on the group start. a group opening on an IDR presents immediately; a rendition without `warmup` is untouched. -## Required - -- [Rust video delivery continuity](/quest/m1/video-continuous.md) - the Rust video signal this keys on - ## Related - [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - trims frames stamped before the keyframe where this trims frames after the start diff --git a/rs/hang/examples/subscribe.rs b/rs/hang/examples/subscribe.rs index 9622449b2e0..f8fdb09e2f7 100644 --- a/rs/hang/examples/subscribe.rs +++ b/rs/hang/examples/subscribe.rs @@ -115,13 +115,17 @@ async fn play(consumer: moq_net::origin::Consumer, announce: moq_net::announce:: moq_mux::container::Consumer::new(track_consumer, moq_mux::catalog::hang::Container::try_from(config)?); // Read frames in latency-bounded presentation order. - while let Some(frame) = ordered.read().await? { - tracing::info!( - timestamp = ?frame.frame.timestamp, - keyframe = frame.frame.keyframe, - bytes = frame.frame.payload.len(), - "received frame" - ); + while let Some(read) = ordered.read().await? { + match read { + moq_mux::container::Read::Frame { frame, .. } => tracing::info!( + timestamp = ?frame.timestamp, + keyframe = frame.keyframe, + bytes = frame.payload.len(), + "received frame" + ), + moq_mux::container::Read::Gap => tracing::info!("media skipped"), + moq_mux::container::Read::End { end, .. } => tracing::info!(?end, "media stopped"), + } } Ok(()) diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 50abb09d301..80e7585b265 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -1,7 +1,8 @@ //! Subscribe to an encoded audio track and emit raw PCM. -use moq_mux::container::Delivery; use std::collections::VecDeque; + +use moq_mux::container::Read; use std::time::{Duration, Instant}; use bytes::Bytes; @@ -67,8 +68,8 @@ impl Options { /// /// The mirror of [`encode::Producer`](crate::encode::Producer): output format / /// sample rate / layout are fixed at construction, and -/// [`read`](Self::read) returns deliveries carrying each frame's continuity -/// and codec activity. +/// [`read`](Self::read) returns [`Frame`]s carrying the codec activity they +/// were decoded from. pub struct Consumer { decoder: Decoder, track: moq_mux::container::Consumer, @@ -85,7 +86,7 @@ pub struct Consumer { next_start: Option, /// Frames decoded and not yet handed back, so a gap's tail can be returned /// ahead of the packet that exposed it. - ready: VecDeque>, + ready: VecDeque, /// Codec activity spans the resampler's buffered output still covers. spans: VecDeque, /// Activity of the last span the output ran past, for the rounding samples the @@ -100,11 +101,10 @@ pub struct Consumer { frames_decoded: usize, /// Logical endpoint carried by an empty legacy frame before terminal packets. end: Option, + /// The group `end` came from: it bounds only the terminal packets that follow it there. + end_group: u64, /// Presentation time of the first decoded terminal frame. terminal_start: Option, - /// Last container playhead generation applied to timeline state. - discontinuity: u64, - continuous: bool, /// The playout delay, fixed or estimated from arrivals. target: Target, /// Where arrival times are measured from. @@ -208,9 +208,8 @@ impl Consumer { delay_trimmed: 0, frames_decoded: 0, end: None, + end_group: 0, terminal_start: None, - discontinuity: 0, - continuous: false, origin: now(), }) } @@ -253,7 +252,6 @@ impl Consumer { } /// Read the next decoded PCM frame, or `None` when the track ends. - /// The returned delivery carries the container's continuity through codec buffering. /// /// [`Frame::activity`] reports whether the packet these samples came from /// coded audio. It describes where the frame begins, so a resampled @@ -265,23 +263,35 @@ impl Consumer { /// side stay anchored to their own packet timeline, so the hole is there to /// see. "Doesn't continue" allows for the quantization the stamps carry, which /// on a millisecond-stamped ingest is most of a millisecond. - pub async fn read(&mut self) -> Result>, Error> { + pub async fn read(&mut self) -> Result, Error> { loop { if let Some(frame) = self.ready.pop_front() { return Ok(Some(frame)); } - let mux_frame = self.track.read().await?; - self.apply_discontinuity()?; - let Some(delivery) = mux_frame else { - if let Some(frame) = self.flush()? { - let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Delivery { frame, continuous }); + let mux_frame = match self.track.read().await? { + Some(Read::Frame { group, frame }) => { + // A later group after a declared end comes from a fresh encoder; the end's + // own group carries the stopped encoder's final packets, trimmed below. + if self.end.is_some() && group != self.end_group { + self.restart()?; + } + frame + } + Some(Read::Gap) => { + self.restart()?; + continue; + } + Some(Read::End { group, end }) => { + self.end = Some(end); + self.end_group = group; + self.frames_decoded = 0; + self.terminal_start = None; + continue; } - return Ok(self.ready.pop_front()); + None => return self.flush(), }; - let mux_frame = delivery.frame; // Every packet the container hands over is an arrival, including one the // decoder is about to refuse, so it is observed before anything can fail. let arrival = jitter::millis(now().duration_since(self.origin)); @@ -289,14 +299,6 @@ impl Consumer { let media = timestamp.value() as f64 * 1000.0 / timestamp.scale().as_u64() as f64; self.target.observe(arrival, media); - if let Some(end) = self.track.end() - && self.end != Some(end) - { - self.end = Some(end); - self.frames_decoded = 0; - self.terminal_start = None; - } - // Undeclared holes are routine: a skipped stalled group, a packet the // decoder refused, an ingest that resynced. Drop every stage's state at // the edge, before the packet after it goes anywhere near the decoder. @@ -310,11 +312,9 @@ impl Consumer { .is_some_and(|next| discontinuous(next, mux_frame.timestamp)) && let Some(frame) = self.gap()? { - let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Delivery { frame, continuous }); + self.ready.push_back(frame); } - self.continuous &= delivery.continuous; let rate = self.decoder.sample_rate(); let epoch = *self.epoch.get_or_insert(mux_frame.timestamp); let delay = self.decoder.delay_remaining(); @@ -408,26 +408,19 @@ impl Consumer { // Queued rather than returned, so a tail drained at a gap earlier in this // same iteration still comes out first. The next turn of the loop pops it. let frame = self.frame(pcm, timestamp, activity)?; - let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Delivery { frame, continuous }); + self.ready.push_back(frame); } } - /// A playhead event re-applies startup delay and skip. The decoder is not reset: + /// Start a new timeline after a gap or past a declared end. The decoder is not reset: /// the next group already starts on a keyframe, and pre-skip is a play-path concern. - fn apply_discontinuity(&mut self) -> Result<(), Error> { - let discontinuity = self.track.discontinuity(); - if discontinuity == self.discontinuity { - return Ok(()); - } - - // Samples already buffered belong to the old span, even when the next - // packet declares a new one. Drain before clearing their timeline labels. + /// + /// Samples the resampler still holds arrived before the break, so they are drained + /// as their own frame first rather than filtered together with what follows it. + fn restart(&mut self) -> Result<(), Error> { if let Some(frame) = self.drain()? { - let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Delivery { frame, continuous }); + self.ready.push_back(frame); } - self.discontinuity = discontinuity; self.next_start = None; self.spans.clear(); self.trailing = Activity::Active; @@ -461,7 +454,7 @@ impl Consumer { Ok(frame) } - /// Drain one span while keeping the resampler available for the next. + /// Drain what the resampler holds while keeping it for what follows. fn drain(&mut self) -> Result, Error> { let Some(resampler) = self.resampler.as_mut() else { return Ok(None); @@ -618,15 +611,16 @@ mod tests { use crate::encode::{Encoder, Input, Options as EncodeOptions, Producer, Settings}; use crate::{Format, Layout}; - #[tokio::test] - async fn continuity_survives_audio_decode_and_a_marker() { - use crate::decode::decoder::tests::{opus_catalog, opus_packets}; + /// What the resampler holds at a marker belongs before it: draining it after the next + /// packet would splice it onto the resumed audio and stamp that late. + #[tokio::test(start_paused = true)] + async fn resampled_tail_stays_before_a_marker() { let broadcast = moq_net::broadcast::Info::new().produce(); let track = broadcast .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) .unwrap(); let subscriber = broadcast.consume(); - let catalog = opus_catalog(moq_mux::codec::opus::Config::new(48_000, 1).with_pre_skip(312)); + let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 44_100, 1); let mut producer = moq_mux::container::Producer::new( track, moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), @@ -636,31 +630,40 @@ mod tests { &catalog, "audio", Options { - max_delay: std::time::Duration::from_secs(10), + max_delay: Duration::from_secs(10), + output: Output { + sample_rate: Some(48_000), + ..Output::default() + }, ..Options::new() }, ) .await .unwrap(); - for (index, payload) in opus_packets(4).into_iter().enumerate() { - if index == 2 { + for index in 0..2 { + if index != 0 { producer.discontinuity().unwrap(); } producer .write(moq_mux::container::Frame { - timestamp: Timestamp::from_micros(index as u64 * 20_000).unwrap(), + timestamp: Timestamp::from_micros(index * 1_000_000).unwrap(), duration: None, - payload, - keyframe: index % 2 == 0, + payload: vec![0u8; 1024 * size_of::()].into(), + keyframe: true, }) .unwrap(); } producer.finish().unwrap(); - let mut flags = Vec::new(); - while let Some(delivery) = consumer.read().await.unwrap() { - flags.push(delivery.continuous); + let mut stamps = Vec::new(); + while let Some(frame) = consumer.read().await.unwrap() { + stamps.push(frame.timestamp.as_micros()); } - assert_eq!(flags, [false, true, false, true]); + assert_eq!(stamps.len(), 4, "each span's body and tail: {stamps:?}"); + assert!( + stamps[1] < 1_000_000, + "the old tail stays before the marker: {stamps:?}" + ); + assert_eq!(stamps[2], 1_000_000, "the resumed span starts on its source timestamp"); } #[tokio::test] @@ -705,7 +708,7 @@ mod tests { producer.write(&Frame::new(data.into(), Timestamp::ZERO)).unwrap(); let frame = consumer.read().await.unwrap().expect("decoded frame"); - let samples = Format::F32.as_interleaved_f32(&frame.frame.data, 2).unwrap(); + let samples = Format::F32.as_interleaved_f32(&frame.data, 2).unwrap(); assert_eq!(samples.len(), (960 - 312) * 2); for pair in samples.as_chunks::<2>().0.iter() { assert_eq!(pair[0], pair[1]); @@ -749,8 +752,8 @@ mod tests { // 312 samples at 48 kHz is 6.5 ms. for (micros, frames) in [(0, 960 - 312), (13_500, 960), (33_500, 960)] { let frame = consumer.read().await.unwrap().expect("decoded frame"); - assert_eq!(frame.frame.timestamp.as_micros(), micros); - assert_eq!(frame.frame.data.len() / size_of::(), frames); + assert_eq!(frame.timestamp.as_micros(), micros); + assert_eq!(frame.data.len() / size_of::(), frames); } } @@ -805,7 +808,7 @@ mod tests { } let first = consumer.read().await.unwrap().expect("decoded frame"); - assert_eq!(first.frame.timestamp.as_micros(), 0); + assert_eq!(first.timestamp.as_micros(), 0); // Continuity, not a fixed number: the second frame starts where the first // one's samples end, whatever they came to. Within a few frames rather than @@ -813,9 +816,9 @@ mod tests { // wobbles around the nominal ratio; a real hole (the samples it held back, or // the startup silence it dropped) is twenty times this tolerance. let second = consumer.read().await.unwrap().expect("decoded frame"); - let first_frames = (first.frame.data.len() / size_of::()) as u128; + let first_frames = (first.data.len() / size_of::()) as u128; let ends_at = first_frames * 1_000_000 / 48_000; - let gap = second.frame.timestamp.as_micros().abs_diff(ends_at); + let gap = second.timestamp.as_micros().abs_diff(ends_at); assert!(gap < 100, "expected the frames to meet, got a {gap} us gap"); } @@ -865,10 +868,10 @@ mod tests { producer.finish().unwrap(); let first = consumer.read().await.unwrap().expect("decoded frame"); - let first_frames = first.frame.data.len() / size_of::(); + let first_frames = first.data.len() / size_of::(); let tail = consumer.read().await.unwrap().expect("flushed tail"); - let tail_frames = tail.frame.data.len() / size_of::(); + let tail_frames = tail.data.len() / size_of::(); // The 142 held-back frames at 44.1 kHz are ~155 at 48 kHz, plus the 69 the // sinc filter still owes: it runs centred, so the end of the track only @@ -877,7 +880,7 @@ mod tests { // It picks up where the first frame's samples ended, within the same few // frames of whole-frame rounding as above. let ends_at = (first_frames as u128) * 1_000_000 / 48_000; - let gap = tail.frame.timestamp.as_micros().abs_diff(ends_at); + let gap = tail.timestamp.as_micros().abs_diff(ends_at); assert!(gap < 100, "expected the tail to meet the body, got a {gap} us gap"); // Together they cover the packet and no more: 1024 frames at 44.1 kHz is @@ -888,62 +891,6 @@ mod tests { assert!(consumer.read().await.unwrap().is_none()); } - #[tokio::test(start_paused = true)] - async fn resampled_continuity_keeps_the_tail_before_a_marker() { - let broadcast = moq_net::broadcast::Info::new().produce(); - let track = broadcast - .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) - .unwrap(); - let subscriber = broadcast.consume(); - let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 44_100, 1); - let mut producer = moq_mux::container::Producer::new( - track, - moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), - ); - let mut consumer = Consumer::new( - &subscriber, - &catalog, - "audio", - Options { - max_delay: Duration::from_secs(10), - output: Output { - sample_rate: Some(48_000), - ..Output::default() - }, - ..Options::new() - }, - ) - .await - .unwrap(); - for index in 0..2 { - if index != 0 { - producer.discontinuity().unwrap(); - } - producer - .write(moq_mux::container::Frame { - timestamp: Timestamp::from_micros(index * 1_000_000).unwrap(), - duration: None, - payload: vec![0u8; 1024 * size_of::()].into(), - keyframe: true, - }) - .unwrap(); - } - producer.finish().unwrap(); - let mut samples = Vec::new(); - while let Some(delivery) = consumer.read().await.unwrap() { - samples.push((delivery.frame.timestamp.as_micros(), delivery.continuous)); - } - assert_eq!( - samples.iter().map(|(_, continuous)| *continuous).collect::>(), - [false, true, false, true] - ); - assert!( - samples[1].0 < 1_000_000, - "the old resampler tail stays before the marker" - ); - assert_eq!(samples[2].0, 1_000_000, "the new span starts on its source timestamp"); - } - #[tokio::test] async fn resampling_keeps_the_activity_boundary_on_its_source() { let mut encoder = Encoder::new(&Settings { @@ -1008,9 +955,9 @@ mod tests { // nothing every other packet. Those must not surface as frames: a frame // with no samples reads as audio arriving, and carries an activity // describing samples that are not there. - assert!(!frame.frame.data.is_empty(), "read returned a frame with no samples"); - if frame.frame.activity.is_dtx() { - actual = Some(frame.frame.timestamp); + assert!(!frame.data.is_empty(), "read returned a frame with no samples"); + if frame.activity.is_dtx() { + actual = Some(frame.timestamp); break; } } @@ -1078,10 +1025,7 @@ mod tests { let mut read = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - read.push(( - frame.frame.timestamp.as_micros(), - frame.frame.data.len() / size_of::(), - )); + read.push((frame.timestamp.as_micros(), frame.data.len() / size_of::())); } read } @@ -1232,9 +1176,9 @@ mod tests { // The output begins with the first packet's samples, so it is stamped and // labelled from that packet. Rewinding from the terminal one instead drops it // most of a second into the future, carrying the DTX label with it. - assert_eq!(frame.frame.timestamp.as_micros(), 0, "held samples moved with the jump"); + assert_eq!(frame.timestamp.as_micros(), 0, "held samples moved with the jump"); assert!( - frame.frame.activity.is_active(), + frame.activity.is_active(), "held samples took the terminal packet's label" ); } @@ -1332,24 +1276,20 @@ mod tests { // shortfall is codec delay, not a hole: without counting it the packet after // every stream start would read as a gap. let first = consumer.read().await.unwrap().expect("decoded frame"); - let frames = first.frame.data.len() / size_of::(); + let frames = first.data.len() / size_of::(); assert!(frames < 960, "the pre-skip should be trimmed, got {frames} frames"); // The hole is real, so codec prediction starts over but stream-level pre-skip // does not. The audio is stamped where the packet says rather than 2.5 ms early. let second = consumer.read().await.unwrap().expect("decoded frame"); - assert_eq!(second.frame.timestamp.as_micros(), 22_500); - assert_eq!( - second.frame.data.len() / size_of::(), - 960, - "pre-skip was reapplied" - ); + assert_eq!(second.timestamp.as_micros(), 22_500); + assert_eq!(second.data.len() / size_of::(), 960, "pre-skip was reapplied"); let third = consumer.read().await.unwrap().expect("decoded frame after gap"); - let second_frames = second.frame.data.len() / size_of::(); + let second_frames = second.data.len() / size_of::(); assert_eq!( - third.frame.timestamp, - advance(second.frame.timestamp, second_frames, 48_000).unwrap() + third.timestamp, + advance(second.timestamp, second_frames, 48_000).unwrap() ); } @@ -1482,7 +1422,7 @@ mod tests { let frame = consumer.read().await.unwrap().expect("decoded frame"); assert!( - frame.frame.data.len() / size_of::() < encoder.frame_size(), + frame.data.len() / size_of::() < encoder.frame_size(), "pre-skip is trimmed" ); // The cold-start prior reads 100 ms, plus one 20 ms packet. @@ -1587,9 +1527,9 @@ mod tests { let first = consumer.read().await.unwrap().expect("first decoded frame"); let second = consumer.read().await.unwrap().expect("second decoded frame"); - let first_frames = first.frame.data.len() / size_of::(); - let expected = advance(first.frame.timestamp, first_frames, 48_000).unwrap(); - assert_eq!(second.frame.timestamp, expected); + let first_frames = first.data.len() / size_of::(); + let expected = advance(first.timestamp, first_frames, 48_000).unwrap(); + assert_eq!(second.timestamp, expected); } #[tokio::test] @@ -1645,8 +1585,8 @@ mod tests { let first = consumer.read().await.unwrap().expect("first decoded frame"); let _second = consumer.read().await.unwrap().expect("second decoded frame"); let resumed = consumer.read().await.unwrap().expect("resumed decoded frame"); - let first_frames = first.frame.data.len() / size_of::(); - let resumed_frames = resumed.frame.data.len() / size_of::(); + let first_frames = first.data.len() / size_of::(); + let resumed_frames = resumed.data.len() / size_of::(); assert!(first_frames < frame_size, "the first epoch trims pre-skip"); assert_eq!( resumed_frames, first_frames, @@ -1697,7 +1637,7 @@ mod tests { let frame = consumer.read().await.unwrap().expect("decoded frame"); assert_eq!( - Format::F32.as_interleaved_f32(&frame.frame.data, 1).unwrap().as_ref(), + Format::F32.as_interleaved_f32(&frame.data, 1).unwrap().as_ref(), samples ); } @@ -1753,8 +1693,8 @@ mod tests { let frame = consumer.read().await.unwrap().expect("decoded frame"); // `as_micros`, not `==`: the CMAF path carries the fmp4 timescale and // `Timestamp`'s equality is structural, so the scales would have to match too. - assert_eq!(frame.frame.timestamp.as_micros(), 0); - let samples = Format::F32.as_interleaved_f32(&frame.frame.data, 2).unwrap(); + assert_eq!(frame.timestamp.as_micros(), 0); + let samples = Format::F32.as_interleaved_f32(&frame.data, 2).unwrap(); assert_eq!(samples.len(), (960 - 312) * 2); } } diff --git a/rs/moq-audio/src/encode/capture.rs b/rs/moq-audio/src/encode/capture.rs index a9462765b4b..eb4f92b9ea5 100644 --- a/rs/moq-audio/src/encode/capture.rs +++ b/rs/moq-audio/src/encode/capture.rs @@ -2193,8 +2193,11 @@ mod tests { async fn read_new(&self, track: &mut Track, seen: &[u64]) -> u64 { loop { - let packet = track.read().await.unwrap().expect("a published packet"); - let timestamp = u64::try_from(packet.frame.timestamp.as_micros()).unwrap(); + let Some(moq_mux::container::Read::Frame { frame: packet, .. }) = track.read().await.unwrap() + else { + continue; + }; + let timestamp = u64::try_from(packet.timestamp.as_micros()).unwrap(); if !seen.contains(×tamp) { return timestamp; } diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index 456b4601522..bc68c6a7034 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -565,6 +565,18 @@ mod tests { use crate::decode::{Consumer as AudioConsumer, Options as DecodeOptions}; use crate::{Activity, Format, Layout}; + /// The next published packet, passing over gaps and ends. + async fn packet(reader: &mut moq_mux::container::Consumer) -> Option + where + F::Error: std::fmt::Debug, + { + loop { + if let moq_mux::container::Read::Frame { frame, .. } = reader.read().await.unwrap()? { + return Some(frame); + } + } + } + #[tokio::test] async fn demand_follows_subscribers_and_closes_with_the_producer() { let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -648,7 +660,7 @@ mod tests { let mut decoded = Vec::new(); while let Some(frame) = audio.read().await.unwrap() { - let pcm = Format::F32.as_interleaved_f32(&frame.frame.data, 1).unwrap(); + let pcm = Format::F32.as_interleaved_f32(&frame.data, 1).unwrap(); decoded.extend_from_slice(&pcm); } assert_eq!(decoded.len(), frames, "terminal padding extended the source"); @@ -699,7 +711,7 @@ mod tests { producer.finish().unwrap(); let mut packets = 0; - while track.read().await.unwrap().is_some() { + while packet(&mut track).await.is_some() { packets += 1; } assert_eq!(packets, 11); @@ -782,8 +794,8 @@ mod tests { producer.reanchor(); producer.finish().unwrap(); - assert!(track.read().await.unwrap().is_some()); - assert!(track.read().await.unwrap().is_none()); + assert!(packet(&mut track).await.is_some()); + assert!(packet(&mut track).await.is_none()); } /// A codec reset starts a new pre-skip interval at the receiver too. @@ -819,7 +831,7 @@ mod tests { producer.write(&full_frame(0)).unwrap(); let first = audio.read().await.unwrap().expect("first packet before the reanchor"); - assert_eq!(first.frame.data.len() / size_of::(), 960 - 312); + assert_eq!(first.data.len() / size_of::(), 960 - 312); producer.reanchor(); producer.write(&full_frame(1_000_000)).unwrap(); @@ -827,8 +839,8 @@ mod tests { let mut resumed_frames = 0; while let Some(frame) = audio.read().await.unwrap() { - if frame.frame.timestamp.as_micros() >= 1_000_000 { - resumed_frames += frame.frame.data.len() / size_of::(); + if frame.timestamp.as_micros() >= 1_000_000 { + resumed_frames += frame.data.len() / size_of::(); } } assert!(resumed_frames > 0, "the reanchored audio still decodes"); @@ -866,8 +878,8 @@ mod tests { for index in 0..100 { producer.write(&pcm_frame(&silence, index * 20_000)).unwrap(); let consumed = consumer.read().await.unwrap().expect("one decoded frame"); - assert_eq!(producer.activity(), consumed.frame.activity); - if consumed.frame.activity.is_dtx() { + assert_eq!(producer.activity(), consumed.activity); + if consumed.activity.is_dtx() { entered_dtx = true; break; } @@ -883,7 +895,7 @@ mod tests { producer.write(&pcm_frame(&active, 2_000_000)).unwrap(); let consumed = consumer.read().await.unwrap().expect("one decoded frame"); assert_eq!(producer.activity(), Activity::Active); - assert_eq!(consumed.frame.activity, Activity::Active); + assert_eq!(consumed.activity, Activity::Active); } // One 20 ms Opus frame at 48 kHz mono is exactly 960 f32 samples, so each @@ -928,8 +940,8 @@ mod tests { producer.reanchor(); } producer.write(frame).unwrap(); - let read = reader.read().await.unwrap().expect("a packet per full frame"); - pts.push(read.frame.timestamp.as_micros()); + let read = packet(&mut reader).await.expect("a packet per full frame"); + pts.push(read.timestamp.as_micros()); } pts } @@ -1118,16 +1130,7 @@ mod tests { producer.write(&pcm_frame(&[0.1; 4 * 1024], anchor_us)).unwrap(); let mut pts = Vec::new(); for _ in 0..4 { - pts.push( - reader - .read() - .await - .unwrap() - .expect("a packet") - .frame - .timestamp - .as_micros(), - ); + pts.push(packet(&mut reader).await.expect("a packet").timestamp.as_micros()); } pts } @@ -1176,8 +1179,8 @@ mod tests { // A second of audio, so the filter's delay is nowhere near the whole write. producer.write(&pcm_frame(&vec![0.1; 44_100], 1_000_000)).unwrap(); - let first = reader.read().await.unwrap().expect("a packet"); - assert_eq!(first.frame.timestamp.as_micros(), 1_000_000); + let first = packet(&mut reader).await.expect("a packet"); + assert_eq!(first.timestamp.as_micros(), 1_000_000); } #[tokio::test] @@ -1228,7 +1231,7 @@ mod tests { producer.write(&full_frame(0)).unwrap(); producer.finish().unwrap(); - assert!(track.read().await.unwrap().is_some()); + assert!(packet(&mut track).await.is_some()); producer.abort(moq_net::Error::Cancel); } diff --git a/rs/moq-audio/src/playback.rs b/rs/moq-audio/src/playback.rs index 884a06a48cf..798bb5984dc 100644 --- a/rs/moq-audio/src/playback.rs +++ b/rs/moq-audio/src/playback.rs @@ -18,7 +18,7 @@ //! let mut sink = engine.sink(input)?; //! //! while let Some(frame) = audio.read().await? { -//! let write = sink.write(&frame.frame.data)?; +//! let write = sink.write(&frame.data)?; //! if write.dropped_sample_frames > 0 { //! eprintln!("dropped {} live audio frames", write.dropped_sample_frames); //! } diff --git a/rs/moq-audio/tests/import.rs b/rs/moq-audio/tests/import.rs index 160f75faeee..a1a5a147e43 100644 --- a/rs/moq-audio/tests/import.rs +++ b/rs/moq-audio/tests/import.rs @@ -39,7 +39,7 @@ async fn ts_surround_opus_decodes() { .expect("decoded frame timed out") .unwrap() { - let (samples, rest) = frame.frame.data.as_chunks::<4>(); + let (samples, rest) = frame.data.as_chunks::<4>(); assert!(rest.is_empty()); for chunk in samples.as_chunks::<6>().0 { for (channel, sample) in energy.iter_mut().zip(chunk) { diff --git a/rs/moq-audio/tests/roundtrip.rs b/rs/moq-audio/tests/roundtrip.rs index af63857020c..34ba2750638 100644 --- a/rs/moq-audio/tests/roundtrip.rs +++ b/rs/moq-audio/tests/roundtrip.rs @@ -76,7 +76,6 @@ async fn opus_round_trip_48k_stereo() { .unwrap() { let pcm: Vec = frame - .frame .data .as_chunks::<4>() .0 @@ -152,7 +151,7 @@ async fn opus_round_trip_44100_s16_resampled() { .expect("decoded frame timed out") .unwrap() { - total_bytes += frame.frame.data.len() as u64; + total_bytes += frame.data.len() as u64; } assert!( total_bytes > 10_000, @@ -193,9 +192,8 @@ async fn pcm_round_trip_is_lossless() { producer.finish().unwrap(); let frame = consumer.read().await.unwrap().unwrap(); - assert_eq!(frame.frame.timestamp, Timestamp::from_micros(123_000).unwrap()); + assert_eq!(frame.timestamp, Timestamp::from_micros(123_000).unwrap()); let decoded: Vec = frame - .frame .data .as_chunks::<4>() .0 @@ -261,7 +259,6 @@ async fn pcm_five_one_reads_back_at_any_layout() { assert_eq!(consumer.layout(), layout); let decoded = consumer.read().await.unwrap().unwrap(); let samples: Vec = decoded - .frame .data .as_chunks::<4>() .0 diff --git a/rs/moq-c/src/audio.rs b/rs/moq-c/src/audio.rs index 5bd108953ba..f13b3991ba0 100644 --- a/rs/moq-c/src/audio.rs +++ b/rs/moq-c/src/audio.rs @@ -272,7 +272,7 @@ impl Audio { }; // Hold the lock only to buffer the frame; release it before the callback. - let frame_id = State::lock().audio.frames.insert(frame.frame)?; + let frame_id = State::lock().audio.frames.insert(frame)?; callback.call(Ok(frame_id)); } } diff --git a/rs/moq-c/src/consume.rs b/rs/moq-c/src/consume.rs index 66c71f2233b..8c0efdfb479 100644 --- a/rs/moq-c/src/consume.rs +++ b/rs/moq-c/src/consume.rs @@ -539,14 +539,16 @@ impl Consume { let frame = tokio::select! { biased; _ = &mut close => return Ok(()), - frame = track.read() => match frame? { - Some(frame) => frame, + read = track.read() => match read? { + Some(moq_mux::container::Read::Frame { frame, .. }) => frame, + // The C ABI carries frames only. + Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. }) => continue, None => return Ok(()), }, }; // Hold the lock only to buffer the frame; release it before the callback. - let frame_id = State::lock().consume.frame.insert(frame.frame)?; + let frame_id = State::lock().consume.frame.insert(frame)?; callback.call(Ok(frame_id)); } } diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 9ba35c89676..8676e459428 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -442,7 +442,7 @@ async fn play_audio( loop { let frame = match consumer.read().await { - Ok(Some(delivery)) => delivery.frame, + Ok(Some(frame)) => frame, Ok(None) => break, // One bad packet is that packet's problem: the decoder stays usable, so // skip it rather than ending playback and taking the video window down diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index a97506f3600..890b1c628e3 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeSet, VecDeque}; use std::sync::{Arc, Mutex}; use std::time::Duration; -use moq_mux::container::Frame; +use moq_mux::container::{Frame, Read}; use tokio::time::Instant; use super::buffer::Buffer; @@ -73,31 +73,34 @@ impl Video { pub async fn run(self, mut track: Track, mut decoder: impl Decoder) -> anyhow::Result<()> { let buffer = Mutex::new(Buffer::default()); let receive = async { - let mut discontinuity = track.discontinuity(); loop { - let frame = track.read().await?; + let read = track.read().await?; let mut buffer = buffer.lock().unwrap(); let before = buffer.generation; - if track.discontinuity() != discontinuity { - discontinuity = track.discontinuity(); - buffer.clear(); - self.presentation.lock().unwrap().video_restarted(); - } - let ended = frame.is_none(); - if let Some(delivery) = frame { - let frame = delivery.frame; - if self - .presentation - .lock() - .unwrap() - .video(frame.timestamp, Instant::now().into_std()) - { - self.output.send(Event::Wake); + let ended = match read { + Some(Read::Frame { frame, .. }) => { + if self + .presentation + .lock() + .unwrap() + .video(frame.timestamp, Instant::now().into_std()) + { + self.output.send(Event::Wake); + } + buffer.push(frame, self.max_delay); + false } - buffer.push(frame, self.max_delay); - } else { - buffer.ended = true; - } + // Applied in delivery order, so the restart lands before the frame after it. + Some(Read::Gap | Read::End { .. }) => { + buffer.clear(); + self.presentation.lock().unwrap().video_restarted(); + false + } + None => { + buffer.ended = true; + true + } + }; if buffer.generation != before { self.frames.lock().unwrap().clear(); self.output.send(Event::Wake); diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 82786c4f3b8..ec1c356720b 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -1173,13 +1173,7 @@ pub(crate) mod tests { let (name, config) = catalog.video.renditions.iter().next().expect("a video rendition"); let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let container = moq_mux::catalog::hang::Container::try_from(config).unwrap(); - let first = Consumer::new(track, container) - .read() - .await - .unwrap() - .expect("a video frame") - .frame - .timestamp; + let first = next_frame(&mut Consumer::new(track, container)).await.timestamp; assert!( first.as_micros() < Duration::from_secs(5).as_micros(), "the feed's own PTS: {first:?}" @@ -1455,11 +1449,22 @@ pub(crate) mod tests { async fn read_frame(consumer: &moq_net::broadcast::Consumer, name: &str) -> Vec { let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let mut reader = Consumer::new(track, Container::Legacy(moq_mux::container::Kind::Data)); - let frame = tokio::time::timeout(Duration::from_secs(1), reader.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), next_frame(&mut reader)) .await - .expect("verbatim read timed out") - .unwrap() - .expect("a published verbatim frame"); - frame.frame.payload.to_vec() + .expect("verbatim read timed out"); + frame.payload.to_vec() + } + + /// The next frame, passing over gaps and ends. + async fn next_frame(reader: &mut Consumer) -> moq_mux::container::Frame + where + C::Error: std::fmt::Debug, + { + loop { + let read = reader.read().await.unwrap().expect("a published frame"); + if let moq_mux::container::Read::Frame { frame, .. } = read { + return frame; + } + } } } diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index 5d3463a85a8..172db6ecc93 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -342,7 +342,7 @@ struct ConsumerInner { impl ConsumerInner { async fn next(&mut self) -> Result, MoqError> { - Ok(self.consumer.read().await?.map(|delivery| delivery.frame.into())) + Ok(self.consumer.read().await?.map(Into::into)) } } diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 0c8fc417b17..753745fc152 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -1101,11 +1101,14 @@ struct Media { impl Media { async fn next(&mut self) -> Result, MoqError> { - self.inner - .read() - .await? - .map(|delivery| media_frame(delivery.frame)) - .transpose() + loop { + match self.inner.read().await? { + Some(moq_mux::container::Read::Frame { frame, .. }) => return media_frame(frame).map(Some), + // The bindings carry frames only. + Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. }) => continue, + None => return Ok(None), + } + } } } diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index c8d05447892..9b9bb29ca6d 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1094,7 +1094,8 @@ impl Pump { // This rendition is being torn down (its run ended, or a catalog update replaced it). _ = cancel.changed() => return None, frame = track.read() => match frame { - Ok(Some(frame)) => { + Ok(Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. })) => {} + Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { // PTS restarts at zero with every pump, so the segment places that zero at the // current running time, or a synced sink would drop a restarted run as late. let segment = reference_ts.is_none().then(|| { @@ -1107,7 +1108,7 @@ impl Pump { ); gst::event::Segment::new(&segment) }); - let buffer = build_buffer(frame.frame, &mut reference_ts, kind); + let buffer = build_buffer(frame, &mut reference_ts, kind); // pad.push() blocks until downstream accepts the buffer (full queues, a // clock-synced sink). block_in_place hands our sibling tasks to another // worker so a stalled downstream can't pin a runtime thread and starve diff --git a/rs/moq-mux/src/codec/opus/import.rs b/rs/moq-mux/src/codec/opus/import.rs index ed290b67831..1abbbed143e 100644 --- a/rs/moq-mux/src/codec/opus/import.rs +++ b/rs/moq-mux/src/codec/opus/import.rs @@ -173,13 +173,13 @@ mod tests { subscriber, crate::catalog::hang::Container::Loc(crate::container::Kind::Data), ); - let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.frame.payload.as_ref(), payload); - assert_eq!(frame.frame.timestamp, Timestamp::from_micros(1_000).unwrap()); + assert_eq!(frame.payload.as_ref(), payload); + assert_eq!(frame.timestamp, Timestamp::from_micros(1_000).unwrap()); import.finish().unwrap(); } diff --git a/rs/moq-mux/src/codec/vp8/import.rs b/rs/moq-mux/src/codec/vp8/import.rs index f8cb4ce667b..0d6321b68a4 100644 --- a/rs/moq-mux/src/codec/vp8/import.rs +++ b/rs/moq-mux/src/codec/vp8/import.rs @@ -206,13 +206,13 @@ mod tests { subscriber, crate::catalog::hang::Container::Loc(crate::container::Kind::Data), ); - let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.frame.payload, keyframe); - assert_eq!(frame.frame.timestamp, Timestamp::from_micros(0).unwrap()); + assert_eq!(frame.payload, keyframe); + assert_eq!(frame.timestamp, Timestamp::from_micros(0).unwrap()); import.finish().unwrap(); } diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 42f4703bc86..51102c62051 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -3,12 +3,17 @@ use std::task::{Poll, ready}; use moq_net::Timestamp; -use super::{Container, Delivery, Frame, TimestampRewind}; +use super::{Container, Frame, Read, TimestampRewind}; /// Media and clean group boundaries in delivery order. pub(crate) enum Event { Frame(Frame), - FrameEnd(Timestamp), + /// An endpoint marker. `opens` when no media precedes it in its group: a declared stop + /// (a marker group or an audio terminal) rather than the previous frame's duration. + FrameEnd { + end: Timestamp, + opens: bool, + }, GroupEnd, } @@ -49,13 +54,12 @@ pub(crate) enum Event { /// subscribing. [`new`](Self::new) inherits that budget, and /// [`set_max_delay`](Self::set_max_delay) changes it mid-stream. /// -/// ## Timeline discontinuities +/// ## Gaps and ends /// -/// A marker group (no decodable frames, one empty payload) is a walk-now discontinuity. -/// A delivered sequence hole is a playhead event unless the boundary is contiguous within -/// 1 ms, and a latency skip or an evicted current group is the same event. -/// [`discontinuity`](Self::discontinuity) is that playhead generation: re-apply startup -/// delay and skip, not a decoder flush. +/// [`read`](Self::read) yields [`Read::Gap`] before the first frame and wherever media +/// will never arrive: a latency skip, an evicted current group, or a sequence hole whose +/// boundary is not contiguous within 1 ms. A marker group (no decodable frames, one empty +/// payload) is the publisher declaring a stop, read as [`Read::End`]; nothing is missing. /// Empty groups (zero objects) mean nothing. A group whose timestamps fall below the live /// edge earlier groups reached is malformed and aborts the track. pub struct Consumer { @@ -95,13 +99,19 @@ pub struct Consumer { // Increments on a declared marker, an unproven delivered hole, and a latency skip. discontinuity: u64, - delivered: Option, // Increments on a declared marker only: the publisher paused or broke its timeline forward. markers: u64, - // Exclusive audio endpoint delivered before terminal codec packets. - end: Option, + // Media went missing since the last item read, so the next one is preceded by a gap. + // Starts set: nothing before the first frame was read. + gap: bool, + + // The item a gap was returned ahead of. + next: Option, + + // The group the last event came from. + group: u64, } /// Two adjacent groups are timeline-contiguous when the next start is within this slack @@ -145,18 +155,16 @@ impl Consumer { floor: None, presented_end: None, discontinuity: 0, - delivered: None, markers: 0, - end: None, + gap: true, + next: None, + group: 0, } } /// A counter that increments at each playhead event: a declared marker group, an /// unproven delivered hole, a latency skip, or an evicted current group. - /// - /// Downstream consumers re-apply startup delay and skip when it changes. It is not a - /// decoder flush: the next group already starts on a keyframe with parameter sets. - pub fn discontinuity(&self) -> u64 { + pub(crate) fn discontinuity(&self) -> u64 { self.discontinuity } @@ -168,44 +176,54 @@ impl Consumer { self.markers } - /// The exclusive audio endpoint delivered before terminal codec packets. - pub fn end(&self) -> Option { - self.end - } - /// Read the next frame from the track. /// /// This method handles timestamp decoding, group ordering, and age management /// automatically. It will skip groups that are too far behind to maintain the /// configured max delay. /// - /// Returns `None` when the track has ended. - /// The first delivery and the first one after a playhead event have `continuous: false`. - /// An undeclared forward timestamp jump alone does not change the flag. - pub async fn read(&mut self) -> Result, F::Error> { + /// Returns `None` when the track has ended. See [`Read`] for the items between frames; + /// an undeclared forward timestamp jump is not a gap, since nothing says media is missing. + pub async fn read(&mut self) -> Result, F::Error> { kio::wait(|waiter| self.poll_read(waiter)).await } + /// The next frame, passing over gaps and ends. + #[cfg(test)] + pub(crate) async fn read_frame(&mut self) -> Result, F::Error> { + loop { + match self.read().await? { + Some(Read::Frame { frame, .. }) => return Ok(Some(frame)), + Some(Read::Gap | Read::End { .. }) => continue, + None => return Ok(None), + } + } + } + /// Poll-based implementation of the read loop. /// /// Uses a single waiter that gets registered on all relevant kio channels, /// avoiding the need for `tokio::select!` or `FuturesUnordered`. - pub fn poll_read(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { + pub fn poll_read(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { + if let Some(read) = self.next.take() { + return Poll::Ready(Ok(Some(read))); + } loop { - match ready!(self.poll_event(waiter))? { - Some(Event::Frame(frame)) => { - let continuous = self.delivered == Some(self.discontinuity); - self.delivered = Some(self.discontinuity); - return Poll::Ready(Ok(Some(Delivery { frame, continuous }))); - } - Some(Event::FrameEnd(end)) => { - if self.format.kind() == super::Kind::Audio { - self.end = Some(end); - } - } - Some(Event::GroupEnd) => continue, + let read = match ready!(self.poll_event(waiter))? { + Some(Event::Frame(frame)) => Read::Frame { + group: self.group, + frame, + }, + Some(Event::FrameEnd { end, opens: true }) => Read::End { group: self.group, end }, + // A trailing endpoint is the last frame's duration, which only the skip logic needs. + Some(Event::FrameEnd { opens: false, .. } | Event::GroupEnd) => continue, None => return Poll::Ready(Ok(None)), + }; + if std::mem::take(&mut self.gap) { + self.next = Some(read); + return Poll::Ready(Ok(Some(Read::Gap))); } + return Poll::Ready(Ok(Some(read))); } } @@ -266,9 +284,10 @@ impl Consumer { if self.live_edge.is_none_or(|(_, high)| ts.as_micros() > high.as_micros()) { self.live_edge = Some((seq, ts)); } + self.group = seq; return Poll::Ready(Ok(Some(Event::Frame(frame)))); } - Poll::Ready(Ok(Some(Event::FrameEnd(end)))) => { + Poll::Ready(Ok(Some(Event::FrameEnd { end, opens }))) => { let seq = group.group.sequence; if self .live_edge @@ -276,7 +295,8 @@ impl Consumer { { self.live_edge = Some((seq, end)); } - return Poll::Ready(Ok(Some(Event::FrameEnd(end)))); + self.group = seq; + return Poll::Ready(Ok(Some(Event::FrameEnd { end, opens }))); } Poll::Ready(Ok(Some(event))) => return Poll::Ready(Ok(Some(event))), // Still blocked on this group, don't skip it yet. @@ -309,7 +329,7 @@ impl Consumer { "current group evicted; skipping to next buffered group" ); // Whatever the group still held is lost, so this is a skip like any other. - self.bump_playhead(); + self.lose(); self.pending.pop_front(); self.current = self.pending.front().map_or(self.current + 1, |g| g.sequence); continue 'read; @@ -386,7 +406,7 @@ impl Consumer { && (finished || max_timestamp.saturating_sub(next_start) >= self.max_delay) { if !pts_contiguous(current_end.or(self.presented_end), next_start) { - self.bump_playhead(); + self.lose(); } self.current = front_sequence; self.note_group_edge(); @@ -411,9 +431,15 @@ impl Consumer { { let hole = !pts_contiguous(current_end.or(self.presented_end), next_start); let had_marker = self.pending.iter().take(new_idx).any(GroupBuffer::marker); + // Buffered frames are thrown away, and `current_end` counts them, so contiguity alone would hide them. + let discarded = self + .pending + .iter() + .take(new_idx) + .any(|group| !group.buffered.is_empty()); self.pending.drain(0..new_idx); - if hole || had_marker { - self.bump_playhead(); + if hole || had_marker || discarded { + self.lose(); } self.markers += u64::from(had_marker); let new_current = self.pending.front().map(|g| g.sequence).unwrap(); @@ -437,7 +463,7 @@ impl Consumer { if let Some(start) = next_start && !pts_contiguous(current_end.or(self.presented_end), start) { - self.bump_playhead(); + self.lose(); } self.current = front_sequence; self.note_group_edge(); @@ -454,7 +480,12 @@ impl Consumer { fn bump_playhead(&mut self) { self.discontinuity += 1; - self.end = None; + } + + // Media this reader will never deliver: the next item is preceded by a gap. + fn lose(&mut self) { + self.bump_playhead(); + self.gap = true; } fn note_group_edge(&mut self) { @@ -586,8 +617,8 @@ impl GroupBuffer { fn poll_read(&mut self, waiter: &kio::Waiter, format: &F) -> Poll, F::Error>> { loop { if self.markers.front().is_some_and(|(index, _)| *index <= self.delivered) { - let (_, end) = self.markers.pop_front().unwrap(); - return Poll::Ready(Ok(Some(Event::FrameEnd(end)))); + let (index, end) = self.markers.pop_front().unwrap(); + return Poll::Ready(Ok(Some(Event::FrameEnd { end, opens: index == 0 }))); } if let Some(frame) = self.buffered.pop_front() { self.delivered += 1; @@ -837,8 +868,8 @@ mod tests { async fn read_all(consumer: &mut Consumer) -> Result, crate::Error> { let mut frames = Vec::new(); loop { - match tokio::time::timeout(Duration::from_millis(200), consumer.read()).await { - Ok(Ok(Some(frame))) => frames.push(frame.frame), + match tokio::time::timeout(Duration::from_millis(200), consumer.read_frame()).await { + Ok(Ok(Some(frame))) => frames.push(frame), Ok(Ok(None)) => break, Ok(Err(e)) => return Err(e), Err(_) => panic!( @@ -919,27 +950,49 @@ mod tests { group.finish().unwrap(); } + /// Every item until the track ends: a frame as its timestamp in µs, a gap, or an end. + async fn items(consumer: &mut Consumer) -> Vec { + let mut items = Vec::new(); + while let Some(read) = consumer.read().await.unwrap() { + items.push(match read { + Read::Frame { frame, .. } => frame.timestamp.as_micros().to_string(), + Read::Gap => "gap".to_string(), + Read::End { end, .. } => format!("end {}", end.as_micros()), + }); + } + items + } + #[tokio::test] - async fn continuity_marks_subscribe_holes_and_markers() { - let mut track = track_producer("continuity", hang::container::track_info(hang::catalog::PRIORITY.audio)); + async fn gaps_mark_the_join_and_holes_while_a_marker_reads_as_an_end() { + let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); let subscription = track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); write_group(&mut track, 0, &[ts(0), ts(10_000)]); write_group(&mut track, 2, &[ts(1_000_000), ts(1_010_000)]); - write_marker_group(&mut track, 3, ts(1_010_000)); + write_marker_group(&mut track, 3, ts(1_020_000)); write_group(&mut track, 4, &[ts(2_000_000), ts(2_010_000)]); track.finish().unwrap(); - let mut flags = Vec::new(); - while let Some(delivery) = reader.read().await.unwrap() { - flags.push(delivery.continuous); - } - assert_eq!(flags, [false, true, false, true, false, true]); + assert_eq!( + items(&mut reader).await, + [ + "gap", + "0", + "10000", + "gap", + "1000000", + "1010000", + "end 1020000", + "2000000", + "2010000" + ] + ); } #[tokio::test] - async fn continuity_marks_a_latency_skip() { - let mut track = track_producer("continuity", hang::container::track_info(hang::catalog::PRIORITY.audio)); + async fn a_latency_skip_is_a_gap() { + let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); let subscription = track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); @@ -947,8 +1000,63 @@ mod tests { write_group(&mut track, 0, &[ts(0)]); write_group(&mut track, 3, &[ts(1_000_000)]); track.finish().unwrap(); - assert!(!reader.read().await.unwrap().unwrap().continuous); - assert!(!reader.read().await.unwrap().unwrap().continuous); + assert_eq!(items(&mut reader).await, ["gap", "0", "gap", "1000000"]); + } + + /// A video group's trailing duration marker is that frame's end, not a declared stop. + #[tokio::test] + async fn a_trailing_duration_marker_is_not_an_end() { + let track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.video)); + let subscription = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Video)); + let wire = Container::Legacy(crate::container::Kind::Video); + for (sequence, start) in [(0, 0), (1, 33_000)] { + let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let frame = |timestamp: u64, payload| Frame { + timestamp: ts(timestamp), + payload, + keyframe: timestamp == start, + duration: None, + }; + wire.write( + &mut group, + &[ + frame(start, Bytes::from_static(&[1])), + frame(start + 33_000, Bytes::new()), + ], + ) + .unwrap(); + group.finish().unwrap(); + } + track.finish().unwrap(); + assert_eq!(items(&mut reader).await, ["gap", "0", "33000"]); + } + + /// An audio terminal opens its group with the endpoint, then the codec's final packets. + #[tokio::test] + async fn an_audio_terminal_reads_its_end_before_the_final_packets() { + let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscription = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); + write_group(&mut track, 0, &[ts(0), ts(20_000)]); + let mut group = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let frame = |timestamp: u64, payload| Frame { + timestamp: ts(timestamp), + payload, + keyframe: false, + duration: None, + }; + Container::Legacy(crate::container::Kind::Audio) + .write( + &mut group, + &[frame(50_000, Bytes::new()), frame(40_000, Bytes::from_static(&[1]))], + ) + .unwrap(); + group.finish().unwrap(); + track.finish().unwrap(); + assert_eq!(items(&mut reader).await, ["gap", "0", "20000", "end 50000", "40000"]); } #[tokio::test] @@ -963,9 +1071,9 @@ mod tests { write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1); } @@ -985,11 +1093,11 @@ mod tests { write_group(&mut track, 4, &[ts(2_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1, "the hole moves the playhead"); assert_eq!(consumer.markers(), 0, "but the timeline carried on"); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(2_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(2_000_000)); assert_eq!(consumer.discontinuity(), 2); assert_eq!(consumer.markers(), 1, "the marker declares a break"); } @@ -1010,8 +1118,8 @@ mod tests { write_group(&mut track, 2, &[ts(1_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000)); assert_eq!(consumer.discontinuity(), 0, "an empty group is not a marker"); } @@ -1028,9 +1136,9 @@ mod tests { write_group(&mut track, 3, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!( consumer.discontinuity(), 1, @@ -1219,13 +1327,13 @@ mod tests { .write(group, &[frame]) .unwrap(); }; - let read = |consumer: &mut Consumer| match consumer.poll_read(&waiter) { - Poll::Ready(Ok(Some(frame))) => Some(frame.frame.timestamp), - Poll::Pending => None, - other => panic!( - "unexpected read: {:?}", - other.map(|r| r.map(|f| f.map(|f| f.frame.timestamp))) - ), + let read = |consumer: &mut Consumer| loop { + match consumer.poll_read(&waiter) { + Poll::Ready(Ok(Some(Read::Frame { frame, .. }))) => break Some(frame.timestamp), + Poll::Ready(Ok(Some(Read::Gap))) => continue, + Poll::Pending => break None, + other => panic!("unexpected read: {other:?}"), + } }; // A 2.5 s group, half arrived, and the next one already 0.5 s in. @@ -1262,7 +1370,7 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(100_000)); write_group(&mut track, 1, &[ts(0)]); track.finish().unwrap(); let err = consumer.read().await.unwrap_err(); @@ -1296,10 +1404,10 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(100_000)); write_group(&mut track, 1, &[ts(100_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(100_000)); } /// A keyframe one frame below the previous group's last frame, but above its start, is an @@ -1312,7 +1420,7 @@ mod tests { // Read the first group before the second arrives: a blocked group that the next one // already covers is skipped for latency, which is not what this checks. for expected in [0, 33_000, 66_000] { - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(expected)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(expected)); } write_group(&mut track, 1, &[ts(50_000), ts(83_000)]); track.finish().unwrap(); @@ -1349,9 +1457,9 @@ mod tests { write_marker_group(&mut track, 1, ts(0)); write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1); } @@ -1361,11 +1469,11 @@ mod tests { let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Audio)); consumer.max_delay = Duration::ZERO; write_group(&mut track, 0, &[ts(0)]); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); write_marker_group(&mut track, 1, ts(0)); write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!( consumer.discontinuity(), 1, @@ -1391,12 +1499,12 @@ mod tests { }], ) .unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); write_marker_group(&mut track, 1, ts(10_000)); write_group(&mut track, 2, &[ts(10_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(10_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(10_000)); assert_eq!( consumer.discontinuity(), 1, @@ -1410,10 +1518,10 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(100_000)); write_group(&mut track, 1, &[ts(200_000), ts(50_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(200_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(200_000)); let err = consumer.read().await.unwrap_err(); assert!(matches!( err, @@ -1499,7 +1607,7 @@ mod tests { group.finish().unwrap(); track.finish().unwrap(); - let frame = consumer.read().await.unwrap().unwrap().frame; + let frame = consumer.read_frame().await.unwrap().unwrap(); assert!(frame.keyframe, "the marker does not consume the first-media slot"); assert!(frame.duration.is_none(), "a leading marker has no previous frame"); } @@ -1686,7 +1794,7 @@ mod tests { let result = tokio::time::timeout(Duration::from_millis(500), async { let mut frames = Vec::new(); while let Ok(Some(frame)) = consumer.read().await { - frames.push(frame.frame); + frames.push(frame); } frames }) @@ -1797,7 +1905,7 @@ mod tests { route.requested_broadcast().await.unwrap().accept(&source); track.demand().used().await.unwrap(); if sequence == 1 { - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); } for sequence in 1..=sequence { write_group(&mut track, sequence, &[ts(sequence * 1_000_000)]); @@ -1825,7 +1933,7 @@ mod tests { matches!(consumer.poll_event(&waiter), Poll::Ready(Ok(Some(Event::GroupEnd)))), "the complete successor still emits its clean boundary" ); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(2_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(2_000_000)); } // ---- Eviction recovery (pause/resume) ---- @@ -1855,7 +1963,7 @@ mod tests { }], ) .unwrap(); - let first = consumer.read().await.unwrap().unwrap().frame; + let first = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(first.timestamp, ts(0)); // A live group arrives far ahead -- sequences 1..4 never come (evicted). The @@ -1872,12 +1980,9 @@ mod tests { .expect("consumer hung on an evicted group / gap") .unwrap() .unwrap(); - assert_eq!( - next.frame.timestamp, - ts(150_000), - "skipped the evicted gap to the live group" - ); - assert!(!next.continuous, "the eviction skip starts a new span"); + assert!(matches!(next, Read::Gap), "the evicted media is a gap"); + let next = consumer.read_frame().await.unwrap().unwrap(); + assert_eq!(next.timestamp, ts(150_000), "skipped the evicted gap to the live group"); } /// A missing (evicted) sequence with a newer group buffered must be skipped once the @@ -1903,7 +2008,7 @@ mod tests { // Reading must reach group 2 across the gap instead of waiting forever for 1. let reached = tokio::time::timeout(Duration::from_secs(1), async { loop { - let frame = consumer.read().await.unwrap().unwrap().frame; + let frame = consumer.read_frame().await.unwrap().unwrap(); if frame.timestamp == ts(200_000) { return; } @@ -1929,8 +2034,8 @@ mod tests { write_group(&mut track, 0, &[ts(0)]); write_group(&mut track, 2, &[ts(200_000)]); - let first = consumer.read().await.unwrap().expect("group 0 frame"); - assert_eq!(first.frame.timestamp, ts(0)); + let first = consumer.read_frame().await.unwrap().expect("group 0 frame"); + assert_eq!(first.timestamp, ts(0)); // Group 0 is done; group 1's stream is still racing. Within the budget the // consumer waits at the gap rather than skipping to group 2. @@ -2084,7 +2189,7 @@ mod tests { track.finish().unwrap(); // The first frame decodes; the malformed second frame must surface as an error. - let first = consumer.read().await; + let first = consumer.read_frame().await; assert!(matches!(first, Ok(Some(_))), "first frame should decode, got {first:?}"); let second = tokio::time::timeout(Duration::from_millis(200), consumer.read()) @@ -2191,8 +2296,8 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } frames }) @@ -2232,7 +2337,7 @@ mod tests { write_group(&mut track, 0, &[ts(0)]); track.finish().unwrap(); - let frame = consumer.read().await.unwrap().unwrap().frame; + let frame = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(frame.timestamp, ts(0)); consumer.set_max_delay(Duration::from_millis(100)); @@ -2275,8 +2380,8 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } frames }) @@ -2336,8 +2441,8 @@ mod tests { let _frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } frames }) @@ -2482,12 +2587,12 @@ mod tests { // group is served, and nothing else arrives. write_group(&mut track, 100, &[ts(3_000_000)]); - let frame = tokio::time::timeout(Duration::from_millis(200), consumer.read()) + let frame = tokio::time::timeout(Duration::from_millis(200), consumer.read_frame()) .await .expect("an unfloored join must not wait out a gap below the served start") .unwrap() .expect("track still live"); - assert_eq!(frame.frame.timestamp, ts(3_000_000)); + assert_eq!(frame.timestamp, ts(3_000_000)); } #[tokio::test] @@ -2666,7 +2771,7 @@ mod tests { track.finish().unwrap(); // Take the first frame only, leaving the second unread. - let first = consumer.read().await.unwrap().unwrap().frame; + let first = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(first.timestamp, ts(0)); // Expiry aborts the finished group while the reader is still inside it. @@ -2741,8 +2846,8 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } frames }) @@ -2798,8 +2903,8 @@ mod tests { track.finish().unwrap(); let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } assert_eq!(frames.len(), 3); @@ -2840,8 +2945,8 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } frames }) @@ -2876,8 +2981,8 @@ mod tests { group1.finish().unwrap(); track.finish().unwrap(); let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } assert_eq!(frames.len(), 2); assert_eq!(consumer.discontinuity(), 0); @@ -2915,8 +3020,8 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } frames }) @@ -2946,14 +3051,14 @@ mod tests { }], ) .unwrap(); - let frame = tokio::time::timeout(Duration::from_secs(1), consumer.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), consumer.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.frame.timestamp, ts(0)); + assert_eq!(frame.timestamp, ts(0)); write_marker(&mut group, ts(15_000)); let event = kio::wait(|waiter| consumer.poll_event(waiter)).await.unwrap(); - assert!(matches!(event, Some(Event::FrameEnd(end)) if end == ts(15_000))); + assert!(matches!(event, Some(Event::FrameEnd { end, opens: false }) if end == ts(15_000))); } } diff --git a/rs/moq-mux/src/container/flv/import_test.rs b/rs/moq-mux/src/container/flv/import_test.rs index 618118ba95f..3f0b3b29349 100644 --- a/rs/moq-mux/src/container/flv/import_test.rs +++ b/rs/moq-mux/src/container/flv/import_test.rs @@ -181,13 +181,10 @@ async fn import_emits_frames() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = decoder.read().await.unwrap().expect("a video frame"); - assert!(frame.frame.keyframe); + let frame = decoder.read_frame().await.unwrap().expect("a video frame"); + assert!(frame.keyframe); // The payload is the length-prefixed NALU, carried through verbatim. - assert_eq!( - frame.frame.payload.as_ref(), - &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00] - ); + assert_eq!(frame.payload.as_ref(), &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00]); } #[tokio::test(start_paused = true)] @@ -210,15 +207,12 @@ async fn public_container_preserves_loc_for_flv() { track, crate::catalog::hang::Container::Loc(crate::container::Kind::Data), ); - let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!( - frame.frame.payload.as_ref(), - &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00] - ); + assert_eq!(frame.payload.as_ref(), &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00]); } /// Bytes split across two `decode` calls still reassemble into whole tags. @@ -510,9 +504,9 @@ async fn import_enhanced_av1() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = decoder.read().await.unwrap().expect("an AV1 frame"); - assert!(frame.frame.keyframe); - assert_eq!(frame.frame.payload.as_ref(), payload); + let frame = decoder.read_frame().await.unwrap().expect("an AV1 frame"); + assert!(frame.keyframe); + assert_eq!(frame.payload.as_ref(), payload); } /// GStreamer 1.28 `fdkaacenc` output at 48 kHz stereo, remuxed to FLV by ffmpeg 9.0.1, whose @@ -644,8 +638,8 @@ async fn import_reports_negative_pts_and_can_resume() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = decoder.read().await.unwrap().expect("the good frame"); - assert_eq!(frame.frame.timestamp.as_millis(), 10); + let frame = decoder.read_frame().await.unwrap().expect("the good frame"); + assert_eq!(frame.timestamp.as_millis(), 10); } #[tokio::test(start_paused = true)] @@ -684,8 +678,8 @@ async fn import_enhanced_hvc1_applies_composition_time() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = decoder.read().await.unwrap().expect("a video frame"); - assert_eq!(frame.frame.timestamp.as_millis(), 17); + let frame = decoder.read_frame().await.unwrap().expect("a video frame"); + assert_eq!(frame.timestamp.as_millis(), 17); } #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/container/fmp4/export.rs b/rs/moq-mux/src/container/fmp4/export.rs index 81094b85080..686107e362c 100644 --- a/rs/moq-mux/src/container/fmp4/export.rs +++ b/rs/moq-mux/src/container/fmp4/export.rs @@ -563,7 +563,7 @@ impl Export { track.pending = Some(frame); break; } - Poll::Ready(Some(Event::FrameEnd(end))) => { + Poll::Ready(Some(Event::FrameEnd { end, .. })) => { if track.is_video && let Some(last) = track.buffer.last_mut() && last.duration.is_none() diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index da0a0111908..ea2b2f7c503 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -1815,7 +1815,7 @@ async fn webvtt_samples_become_timed_text_with_source_metadata() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap().frame; + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!(frame.timestamp.as_micros(), 250_000); assert_eq!( &frame.payload[..], @@ -1885,7 +1885,7 @@ async fn webvtt_kind_and_lone_cr_terminators() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap().frame; + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!( &frame.payload[..], b"WEBVTT\nKind: captions\n\n00:00:00.250 --> 00:00:01.500\nHello\nworld\n\n" @@ -1996,6 +1996,6 @@ async fn webvtt_sub_millisecond_cue_stays_positive() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap().frame; + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!(&frame.payload[..], b"WEBVTT\n\n00:00:00.250 --> 00:00:00.251\nhi\n\n"); } diff --git a/rs/moq-mux/src/container/group.rs b/rs/moq-mux/src/container/group.rs index 8ac2d9052b2..31fcd7d84e4 100644 --- a/rs/moq-mux/src/container/group.rs +++ b/rs/moq-mux/src/container/group.rs @@ -144,11 +144,11 @@ mod tests { subscription, Hang::new(&config, crate::container::Kind::Data).unwrap(), ); - let first = live.read().await.unwrap().unwrap(); - assert!(first.frame.payload.is_empty()); - assert!(first.frame.keyframe); - assert_eq!(live.read().await.unwrap().unwrap().frame.payload, b"data".as_slice()); - assert!(live.read().await.unwrap().is_none()); + let first = live.read_frame().await.unwrap().unwrap(); + assert!(first.payload.is_empty()); + assert!(first.keyframe); + assert_eq!(live.read_frame().await.unwrap().unwrap().payload, b"data".as_slice()); + assert!(live.read_frame().await.unwrap().is_none()); let group = consumer.track("data").unwrap().fetch_group(0, None).await.unwrap(); let mut fetched = GroupConsumer::new(group, Hang::new(&config, crate::container::Kind::Data).unwrap()); assert!(fetched.read().await.unwrap().unwrap().payload.is_empty()); diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index 0310016ee3a..9a1e8f080bb 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -263,12 +263,12 @@ async fn public_container_preserves_loc_for_mkv() { track, crate::catalog::hang::Container::Loc(crate::container::Kind::Data), ); - let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.frame.payload.as_ref(), payload); + assert_eq!(frame.payload.as_ref(), payload); } #[test] @@ -683,7 +683,7 @@ async fn subtitle_languages_and_empty_tracks_are_published() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap().frame; + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!( &frame.payload[..], b"WEBVTT\n\n00:00:00.250 --> 00:00:01.500\nHello\n\n" @@ -730,7 +730,7 @@ async fn subtitle_duration_survives_as_a_timed_cue() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap().frame; + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!(frame.timestamp.as_micros(), 250_000); assert_eq!( &frame.payload[..], @@ -786,7 +786,7 @@ async fn webvtt_preserves_cue_metadata_and_relative_timestamps() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap().frame; + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!(&frame.payload[..], b"WEBVTT\n\nSTYLE\n::cue { color: lime; }\n\nNOTE source\n\ncue-1\n00:00:00.250 --> 00:00:01.500 align:start\nHello <00:00:00.750>world\n\n"); } @@ -904,11 +904,11 @@ async fn laced_opus_frames() { let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::try_from(config).unwrap()); for (index, packet) in packets.iter().enumerate() { - let frame = reader.read().await.unwrap().expect("one frame per packet").frame; + let frame = reader.read_frame().await.unwrap().expect("one frame per packet"); assert_eq!(frame.payload.as_ref(), *packet, "{lacing:?}, grouped={grouped}"); assert_eq!(frame.timestamp.as_micros(), 107_000 + index as u128 * 20_000); } - assert!(reader.read().await.unwrap().is_none()); + assert!(reader.read_frame().await.unwrap().is_none()); } } } diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 77670d47115..c3ebc30641b 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -45,13 +45,28 @@ pub enum Kind { Data, } -/// A delivered frame and whether it continues the previous delivery. +/// One item of a [`Consumer`] read, in delivery order. #[derive(Clone, Debug)] -pub struct Delivery { - /// The encoded or decoded media frame. - pub frame: T, - /// False on the first frame and after a playhead discontinuity. - pub continuous: bool, +pub enum Read { + /// A media frame from `group`. + Frame { + /// The group's sequence number. Only equality means anything: numbers may skip. + group: u64, + frame: Frame, + }, + /// Media before the next item will never reach this reader: it joined mid-track, or a + /// group was skipped, evicted, or never arrived. A decoder resumes from here without + /// the references or state that media built. + Gap, + /// The publisher declared that its media stops at `end`: it paused, switched source, or + /// finished. Frames that follow from the same `group` are the stopped encoder's final + /// packets (an audio codec's drain), whose output at or past `end` is padding. The next + /// group starts fresh. + End { + /// The group's sequence number. Only equality means anything: numbers may skip. + group: u64, + end: moq_net::Timestamp, + }, } /// A decoded media frame: timestamp, payload bytes, keyframe flag. diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index 02068a33af8..638215537ab 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -1687,8 +1687,8 @@ mod tests { let mut consumer = crate::container::Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Video)); let mut timestamps = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - timestamps.push(frame.frame.timestamp.as_micros()); + while let Some(frame) = consumer.read_frame().await.unwrap() { + timestamps.push(frame.timestamp.as_micros()); } assert_eq!(timestamps, [1_000_000, 1_040_000, 1_050_000]); } diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 9c25239af38..2fcc76aad27 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -315,7 +315,7 @@ impl ExportSource { loop { match ready!(self.poll_event(waiter))? { Some(Event::Frame(frame)) => return Poll::Ready(Ok(Some(frame))), - Some(Event::GroupEnd | Event::FrameEnd(_)) => continue, + Some(Event::GroupEnd | Event::FrameEnd { .. }) => continue, None => return Poll::Ready(Ok(None)), } } diff --git a/rs/moq-mux/src/container/test_util.rs b/rs/moq-mux/src/container/test_util.rs index 05c7a80c9e3..9c65a3995eb 100644 --- a/rs/moq-mux/src/container/test_util.rs +++ b/rs/moq-mux/src/container/test_util.rs @@ -152,12 +152,12 @@ pub(crate) async fn published( let track = consumer.track(&name).unwrap().subscribe(replay).await.unwrap(); let mut reader = crate::container::Consumer::new(track, container); let mut timestamps = Vec::new(); - while let Some(frame) = tokio::time::timeout(std::time::Duration::from_secs(5), reader.read()) + while let Some(frame) = tokio::time::timeout(std::time::Duration::from_secs(5), reader.read_frame()) .await .expect("the importer finished its tracks") .unwrap() { - timestamps.push(frame.frame.timestamp.as_micros()); + timestamps.push(frame.timestamp.as_micros()); } out.insert(name, timestamps); } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index ca005988f6e..b0a68d6dff3 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -1305,12 +1305,12 @@ async fn export_scte35_roundtrip() { .unwrap(); let mut scte_reader = crate::container::Consumer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); let frame = scte_reader - .read() + .read_frame() .await .unwrap() .expect("no SCTE-35 frame after round-trip"); assert_eq!( - frame.frame.payload.as_ref(), + frame.payload.as_ref(), CUE, "SCTE-35 section did not round-trip byte-for-byte" ); @@ -1424,12 +1424,12 @@ async fn export_pes_verbatim_roundtrip() { .unwrap(); let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); let frame = reader - .read() + .read_frame() .await .unwrap() .expect("no verbatim PES frame after round-trip"); assert_eq!( - frame.frame.payload.as_ref(), + frame.payload.as_ref(), PAYLOAD, "verbatim PES payload round-trips byte-for-byte" ); @@ -1502,9 +1502,9 @@ async fn read_frames(consumer: &moq_net::broadcast::Consumer, name: &str, kind: .unwrap(); let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy(kind)); let mut frames = Vec::new(); - while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { + while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { let Some(frame) = res.unwrap() else { break }; - frames.push(frame.frame.payload.to_vec()); + frames.push(frame.payload.to_vec()); } frames } @@ -2083,9 +2083,9 @@ async fn read_cues(consumer: &moq_net::broadcast::Consumer, name: &str) -> Vec<( .unwrap(); let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); let mut cues = Vec::new(); - while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { + while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { let Some(frame) = res.unwrap() else { break }; - cues.push((frame.frame.payload.to_vec(), frame.frame.timestamp)); + cues.push((frame.payload.to_vec(), frame.timestamp)); } cues } diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index bf4ad33e814..36a4cd7fc5f 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -3885,13 +3885,13 @@ pub(super) mod test { .await .unwrap(); let mut reader = Consumer::new(track, Container::Legacy(crate::container::Kind::Data)); - let frame = tokio::time::timeout(Duration::from_secs(1), reader.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), reader.read_frame()) .await .expect("cue read timed out") .unwrap() .expect("a published cue frame"); assert_eq!( - &frame.frame.payload[..], + &frame.payload[..], &CUE[..], "verbatim splice_info_section after upgrade" ); @@ -4554,8 +4554,8 @@ pub(super) mod test { crate::catalog::hang::Container::Legacy(crate::container::Kind::Audio), ); let mut frames = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { - frames.push(frame.frame); + while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { + frames.push(frame); } frames } @@ -4608,8 +4608,8 @@ pub(super) mod test { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let mut frames = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { - frames.push(frame.frame); + while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { + frames.push(frame); } frames } @@ -5432,7 +5432,7 @@ pub(super) mod test { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let published = tokio::time::timeout(Duration::from_millis(50), reader.read()).await; + let published = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await; assert!( matches!(published, Ok(Ok(Some(_)))), "a PID that never published held the catalog shut: {published:?}" @@ -5822,18 +5822,18 @@ pub(super) mod test { .await .unwrap(); let mut reader = Consumer::new(track, Container::Legacy(crate::container::Kind::Data)); - let frame = tokio::time::timeout(Duration::from_secs(1), reader.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), reader.read_frame()) .await .expect("cue read timed out") .unwrap() .expect("a published cue frame"); - assert_eq!(&frame.frame.payload[..], &CUE[..], "verbatim splice_info_section"); - assert_ne!(frame.frame.timestamp, Timestamp::ZERO, "cue must not stamp zero"); + assert_eq!(&frame.payload[..], &CUE[..], "verbatim splice_info_section"); + assert_ne!(frame.timestamp, Timestamp::ZERO, "cue must not stamp zero"); // The legacy container normalizes the wire timestamp to microseconds, so compare the // instant (not the raw scale) against the 90 kHz media clock the cue was stamped with. assert_eq!( - Duration::from(frame.frame.timestamp), + Duration::from(frame.timestamp), Duration::from(clock), "cue stamped with the video media clock" ); @@ -6008,16 +6008,12 @@ pub(super) mod test { .await .unwrap(); let mut reader = Consumer::new(track, Container::Legacy(crate::container::Kind::Data)); - let frame = tokio::time::timeout(Duration::from_secs(1), reader.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), reader.read_frame()) .await .expect("verbatim read timed out") .unwrap() .expect("a published verbatim frame"); - assert_eq!( - &frame.frame.payload[..], - &payload[..], - "verbatim PES payload round-trips" - ); + assert_eq!(&frame.payload[..], &payload[..], "verbatim PES payload round-trips"); } /// A verbatim-only program retains the PES clock while waiting for rate measurement or EOF. @@ -6103,9 +6099,10 @@ pub(super) mod test { crate::catalog::hang::Container::Legacy(crate::container::Kind::Audio), ); let mut frames = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await + while let Ok(Ok(Some(frame))) = + tokio::time::timeout(std::time::Duration::from_millis(50), reader.read_frame()).await { - frames.push(frame.frame); + frames.push(frame); } (frames, reader.discontinuity()) } @@ -6711,9 +6708,9 @@ pub(super) mod test { let track = consumer.track(name).unwrap().subscribe(subscription).await.unwrap(); let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy(kind)); let mut frames = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { - if !frame.frame.payload.is_empty() { - frames.push(frame.frame); + while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { + if !frame.payload.is_empty() { + frames.push(frame); } } frames diff --git a/rs/moq-mux/src/container/ts/import_test.rs b/rs/moq-mux/src/container/ts/import_test.rs index 863033a02b7..5a31923c29b 100644 --- a/rs/moq-mux/src/container/ts/import_test.rs +++ b/rs/moq-mux/src/container/ts/import_test.rs @@ -96,13 +96,13 @@ async fn public_container_preserves_loc_for_ts() { track, crate::catalog::hang::Container::Loc(crate::container::Kind::Data), ); - let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert!(frame.frame.keyframe); - assert!(!frame.frame.payload.is_empty()); + assert!(frame.keyframe); + assert!(!frame.payload.is_empty()); } /// The Kyrion capture is H.264 1080i with B-frames (open-GOP, ~5-frame reorder despite only @@ -262,9 +262,9 @@ async fn import_opus_frames() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let mut frames = Vec::new(); - while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { + while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read_frame()).await { let Some(frame) = res.unwrap() else { break }; - frames.push(frame.frame.payload); + frames.push(frame.payload); } assert!(frames.len() > 5, "expected a run of Opus packets, got {}", frames.len()); @@ -541,14 +541,13 @@ async fn survives_midstream_join() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let mut frames = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { + while let Ok(Ok(Some(frame))) = + tokio::time::timeout(std::time::Duration::from_millis(50), reader.read_frame()).await + { frames.push(frame); } assert_eq!(frames.len(), 1, "expected only the post-join IDR, got {}", frames.len()); - assert!( - frames[0].frame.keyframe, - "the first surviving frame must be the keyframe" - ); + assert!(frames[0].keyframe, "the first surviving frame must be the keyframe"); } /// A real Ateme Kyrion broadcast captured mid-stream with `nc`, so it opens dirty: @@ -598,8 +597,10 @@ async fn kyrion_dirtystart_extracts_real_cues() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let mut cues = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { - cues.push((frame.frame.payload.to_vec(), frame.frame.timestamp)); + while let Ok(Ok(Some(frame))) = + tokio::time::timeout(std::time::Duration::from_millis(50), reader.read_frame()).await + { + cues.push((frame.payload.to_vec(), frame.timestamp)); } assert_eq!(cues.len(), 6, "expected the six real splice_inserts"); assert!( diff --git a/rs/moq-mux/src/container/ts/programs.rs b/rs/moq-mux/src/container/ts/programs.rs index 10e13f53cb5..5324bd19e8f 100644 --- a/rs/moq-mux/src/container/ts/programs.rs +++ b/rs/moq-mux/src/container/ts/programs.rs @@ -376,16 +376,17 @@ mod test { let (name, config) = renditions[0]; let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let container = Container::try_from(config).unwrap(); - let frame = Consumer::new(track, container).read().await.unwrap().expect("a frame"); - assert!( - fills.contains(&frame.frame.payload[4]), - "{path} carries only its own program" - ); + let frame = Consumer::new(track, container) + .read_frame() + .await + .unwrap() + .expect("a frame"); + assert!(fills.contains(&frame.payload[4]), "{path} carries only its own program"); let skew = Duration::from_secs(2); let wall = catalog .clock .expect("the catalog advertises a clock") - .wall_clock(frame.frame.timestamp) + .wall_clock(frame.timestamp) .unwrap(); assert!( before - skew <= wall && wall <= after + skew, diff --git a/rs/moq-mux/src/import/track.rs b/rs/moq-mux/src/import/track.rs index 208b58d8fed..58582eb5059 100644 --- a/rs/moq-mux/src/import/track.rs +++ b/rs/moq-mux/src/import/track.rs @@ -1050,13 +1050,13 @@ mod tests { .decode(&payload, Some(Timestamp::from_micros(1_000).unwrap())) .unwrap(); - let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.frame.payload, payload); - assert_eq!(frame.frame.timestamp, Timestamp::from_micros(1_000).unwrap()); + assert_eq!(frame.payload, payload); + assert_eq!(frame.timestamp, Timestamp::from_micros(1_000).unwrap()); import.finish().unwrap(); } @@ -1378,14 +1378,14 @@ mod tests { .decode(&au, Some(Timestamp::from_micros(1_000).unwrap())) .unwrap(); - let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert!(frame.frame.keyframe, "an IDR access unit must open a group"); - assert_eq!(frame.frame.payload, au, "the payload must pass through verbatim"); - assert_eq!(frame.frame.timestamp, Timestamp::from_micros(1_000).unwrap()); + assert!(frame.keyframe, "an IDR access unit must open a group"); + assert_eq!(frame.payload, au, "the payload must pass through verbatim"); + assert_eq!(frame.timestamp, Timestamp::from_micros(1_000).unwrap()); import.finish().unwrap(); } diff --git a/rs/moq-rtc/src/codec/mod.rs b/rs/moq-rtc/src/codec/mod.rs index 5b3a6c4f0ff..20d24b7a712 100644 --- a/rs/moq-rtc/src/codec/mod.rs +++ b/rs/moq-rtc/src/codec/mod.rs @@ -230,17 +230,20 @@ impl Track { /// Pull the next RTP-ready frame. Returns `None` when the track ends. pub async fn next(&mut self) -> Result> { loop { - let Some(frame) = self.consumer.read().await? else { - return Ok(None); + let frame = match self.consumer.read().await? { + Some(moq_mux::container::Read::Frame { frame, .. }) => frame, + // RTP sequence numbers and timestamps already show the hole to a receiver. + Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. }) => continue, + None => return Ok(None), }; let payload = match &self.convert { - TrackConvert::Passthrough => frame.frame.payload, + TrackConvert::Passthrough => frame.payload, TrackConvert::LengthPrefixed { length_size, keyframe_prefix, } => { - let prefix = frame.frame.keyframe.then(|| keyframe_prefix.as_ref()); - moq_mux::codec::annexb::from_length_prefixed(&frame.frame.payload, *length_size, prefix) + let prefix = frame.keyframe.then(|| keyframe_prefix.as_ref()); + moq_mux::codec::annexb::from_length_prefixed(&frame.payload, *length_size, prefix) .map_err(moq_mux::Error::from)? } }; @@ -248,7 +251,7 @@ impl Track { continue; } return Ok(Some(PacketizedFrame { - timestamp_us: frame.frame.timestamp.as_micros() as u64, + timestamp_us: frame.timestamp.as_micros() as u64, payload, })); } diff --git a/rs/moq-srt/src/ts.rs b/rs/moq-srt/src/ts.rs index c5a01ef9d09..8af41f002de 100644 --- a/rs/moq-srt/src/ts.rs +++ b/rs/moq-srt/src/ts.rs @@ -647,12 +647,11 @@ mod tests { let track = broadcast.track(name.as_str()).unwrap().subscribe(None).await.unwrap(); let mut reader = moq_mux::container::Consumer::new(track, Container::Legacy(moq_mux::container::Kind::Data)); - let cue = timeout(Duration::from_secs(5), reader.read()) + let cue = timeout(Duration::from_secs(5), next_frame(&mut reader)) .await .expect("cue read timed out") - .unwrap() .expect("a published cue section"); - let expected_cue = cue.frame.payload; + let expected_cue = cue.payload; assert_eq!(expected_cue[0], 0xFC, "a verbatim splice_info_section (table_id 0xFC)"); let mut subscriber = Subscriber::new(&origin.consume(), "ingest", Options::default()) @@ -696,18 +695,24 @@ mod tests { .expect("SRT egress preserves the SCTE-35 track"); let track = roundtrip_consumer.track(name).unwrap().subscribe(None).await.unwrap(); let mut reader = moq_mux::container::Consumer::new(track, Container::Legacy(moq_mux::container::Kind::Data)); - let cue = timeout(Duration::from_secs(5), reader.read()) + let cue = timeout(Duration::from_secs(5), next_frame(&mut reader)) .await .expect("round-trip cue read timed out") - .unwrap() .expect("SRT egress preserves a cue section"); + assert_eq!(cue.payload[0], 0xFC, "the round-trip cue is a splice_info_section"); assert_eq!( - cue.frame.payload[0], 0xFC, - "the round-trip cue is a splice_info_section" - ); - assert_eq!( - cue.frame.payload, expected_cue, + cue.payload, expected_cue, "SRT egress preserves the complete cue section" ); } + + /// The next frame, passing over gaps and ends. + async fn next_frame(reader: &mut moq_mux::container::Consumer) -> Option { + loop { + match reader.read().await.unwrap()? { + moq_mux::container::Read::Frame { frame, .. } => return Some(frame), + moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. } => continue, + } + } + } } diff --git a/rs/moq-video/src/capture/wgc.rs b/rs/moq-video/src/capture/wgc.rs index d426fc0b6fb..f2d44221d95 100644 --- a/rs/moq-video/src/capture/wgc.rs +++ b/rs/moq-video/src/capture/wgc.rs @@ -327,8 +327,12 @@ mod tests { let (surface, timestamp) = delivery.next(now).unwrap().unwrap(); let frame = crate::Frame::new(crate::Surface::I420(surface.clone()), timestamp); producer.publish(&encoder.encode(&frame).unwrap()).unwrap(); - let received = subscriber.read().await.unwrap().unwrap(); - assert_eq!(received.frame.timestamp.as_micros(), u128::from(step) * 20_000); + let received = loop { + if let moq_mux::container::Read::Frame { frame, .. } = subscriber.read().await.unwrap().unwrap() { + break frame; + } + }; + assert_eq!(received.timestamp.as_micros(), u128::from(step) * 20_000); } } diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 4ecf60d516a..327e3988017 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -3,6 +3,7 @@ use std::collections::VecDeque; use hang::catalog::VideoConfig; +use moq_mux::container::Read; use super::decoder::Config; use super::sink::Sink; @@ -85,8 +86,6 @@ pub struct Consumer { pending: VecDeque, /// Whether the ended track's decoder has already been drained. drained: bool, - /// Last container playhead generation observed. - discontinuity: u64, } impl Consumer { @@ -140,7 +139,6 @@ impl Consumer { track, pending: VecDeque::new(), drained: false, - discontinuity: 0, }) } @@ -166,14 +164,13 @@ impl Consumer { return Ok(None); } - let mux_frame = self.track.read().await?; - let discontinuity = self.track.discontinuity(); - if discontinuity != self.discontinuity { - // A playhead event re-applies startup delay and skip; the next group - // already starts on a keyframe with parameter sets, so the decoder is - // not flushed. - self.discontinuity = discontinuity; - } + let mux_frame = match self.track.read().await? { + Some(Read::Frame { frame, .. }) => Some(frame), + // The next group starts on a keyframe with parameter sets, so neither a + // gap nor a declared end flushes the decoder. + Some(Read::Gap | Read::End { .. }) => continue, + None => None, + }; let Some(mux_frame) = mux_frame else { // The flag goes up only once the tail is in hand, so a read @@ -194,11 +191,7 @@ impl Consumer { self.pending.extend( self.decoder - .decode( - mux_frame.frame.payload, - mux_frame.frame.timestamp, - mux_frame.frame.keyframe, - ) + .decode(mux_frame.payload, mux_frame.timestamp, mux_frame.keyframe) .await?, ); } diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index 7bfd320c410..5ce8d34910e 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -598,14 +598,19 @@ mod tests { .unwrap(); let mut track = moq_mux::container::Consumer::new(track, moq_mux::catalog::hang::Container::try_from(config).unwrap()); - let sample = track.read().await.unwrap().expect("an imported sample"); - assert!(sample.frame.keyframe); + let sample = loop { + let read = track.read().await.unwrap().expect("an imported sample"); + if let moq_mux::container::Read::Frame { frame, .. } = read { + break frame; + } + }; + assert!(sample.keyframe); let decoder = super::Decoder::new(config, &decode_config(super::Kind::Named(probe::BUFFERED_NAME.into()))).unwrap(); let access_unit = decoder .conversion - .access_unit(&sample.frame.payload, sample.frame.keyframe) + .access_unit(&sample.payload, sample.keyframe) .unwrap(); let mut buf = access_unit.clone(); let mut nals = moq_mux::codec::annexb::NalIterator::new(&mut buf); diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index de8a7aa3227..3241d3c096a 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -1208,8 +1208,12 @@ mod tests { } async fn read(track: &mut moq_mux::container::Consumer) -> u64 { - let frame = track.read().await.unwrap().expect("a published frame"); - u64::try_from(frame.frame.timestamp.as_micros()).unwrap() + loop { + let read = track.read().await.unwrap().expect("a published frame"); + if let moq_mux::container::Read::Frame { frame, .. } = read { + return u64::try_from(frame.timestamp.as_micros()).unwrap(); + } + } } /// Read the next frame not already in `seen`: a resubscription replays retained groups first. From c5817e3cfff5240ea418338bd2797911cf700bdc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 19:59:38 -0700 Subject: [PATCH 07/10] fix(interop): read the frame item in the JS native subscriber Co-Authored-By: Claude Opus 5.5 --- test/interop/clients/js-native/subscribe.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/interop/clients/js-native/subscribe.ts b/test/interop/clients/js-native/subscribe.ts index 4a11e8f3677..b7b3b17f982 100644 --- a/test/interop/clients/js-native/subscribe.ts +++ b/test/interop/clients/js-native/subscribe.ts @@ -129,7 +129,7 @@ async function run(): Promise { for (;;) { const next = await consumer.next(); if (!next) break; - const bytes = next.frame?.payload.byteLength ?? 0; + const bytes = next.kind === "frame" ? next.frame.payload.byteLength : 0; if (bytes > 0) { // The harness judges success by this marker, not the exit code: the // @moq/web-transport NAPI addon can segfault during the runtime's exit From be9337e8f92bf1a7e91d36624cdaf4215a026621 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 20:17:42 -0700 Subject: [PATCH 08/10] fix(hang): read no gap when a retired group lost nothing Retiring a fully delivered group, or one holding only a trailing endpoint, for the max delay read as a gap. Only unread media, a shed marker group, or a broken timeline is a gap, matching the Rust skip path. Co-Authored-By: Claude Opus 5.5 --- js/hang/src/container/consumer.test.ts | 47 ++++++++++++++++++++++++++ js/hang/src/container/consumer.ts | 13 +++---- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index a2c740a4002..70aa40ec9bc 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -965,6 +965,53 @@ test("Consumer accepts open-GOP leading pictures above the previous group", asyn consumer.close(); }); +// A delivered group still awaiting its done result holds no unread media, so retiring it for +// the max delay loses nothing. +test("Consumer reads no gap when retiring a fully delivered group", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 150 as Time.Milli }); + + writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); + await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + + writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); + await settle(); + + // Read directly: a gap here would fail the frame check. + expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); + + consumer.close(); +}); + +// An unread trailing endpoint carries no media, so retiring its group loses nothing. +test("Consumer reads no gap when retiring a group holding only its endpoint", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("video"), maxDelay: 150 as Time.Milli }); + + const group = new Group.Producer(0); + group.writeFrame({ payload: encodeLegacy(0 as Time.Micro), timestamp: Time.Timestamp.now() }); + group.writeFrame({ + payload: encodeLegacyFrame(33_000 as Time.Micro, new Uint8Array()), + timestamp: Time.Timestamp.now(), + }); + group.close(); + track.writeGroup(group); + await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + + writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); + await settle(); + + expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); + + consumer.close(); +}); + // --- Buffered signal --- test("Consumer buffered signal updates as frames arrive", async () => { diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index a0670f027e3..1351bdfe1c0 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -360,13 +360,14 @@ export class Consumer { this.#groups.shift(); this.#active = this.#groups[0]?.consumer.sequence; - const nextStart = this.#groups[0]?.frames.at(0)?.timestamp ?? this.#groups[0]?.end; + const next = this.#groups[0]; + const nextStart = next?.frames.at(0)?.timestamp ?? next?.end; const marker = !first.empty && !first.media; - // Buffered frames are thrown away, and `first.end` counts them, so contiguity alone would hide them. - const discarded = first.frames.length > 0; - if (marker || discarded || !ptsContiguous(first.end ?? this.#presentedEnd, nextStart)) { - this.#lost = true; - } + // Unread media is thrown away, and `first.end` counts it, so contiguity alone would hide it. + // A trailing endpoint carries no media, so leaving it unread discards nothing. + const discarded = first.frames.some((frame) => this.#format.end?.(frame) === undefined); + const continued = continues(first, next) || ptsContiguous(first.end ?? this.#presentedEnd, nextStart); + if (marker || discarded || !continued) this.#lost = true; first.consumer.close(); first.frames.length = 0; skipped++; From 552b23bcb89a009327cae0f476c9cb54d81c388c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 22:05:34 -0700 Subject: [PATCH 09/10] fix(mux): read a joined marker's end, and keep gaps for unfinished retired groups Rust startup now adopts the first group carrying any wire frame, so joining a paused track whose only served group is a marker reads the gap and end instead of waiting for media. JS retirement for the max delay only treats an adjacent successor as continuous once the retired group finished cleanly; a truncated or still-open group may have lost its tail. Co-Authored-By: Claude Opus 5.5 --- js/hang/src/container/consumer.test.ts | 47 ++++++++++++++++++++++++++ js/hang/src/container/consumer.ts | 8 +++-- rs/moq-mux/src/container/consumer.rs | 31 ++++++++++++++--- 3 files changed, 79 insertions(+), 7 deletions(-) diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 70aa40ec9bc..8d9a9f8f4c1 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -1012,6 +1012,53 @@ test("Consumer reads no gap when retiring a group holding only its endpoint", as consumer.close(); }); +// A truncated group lost its tail, so retiring it for the max delay is a gap even when the +// next group is adjacent. +test("Consumer reads a gap when retiring a truncated group", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 150 as Time.Milli }); + + const group = new Group.Producer(0); + group.writeFrame({ payload: encodeLegacy(0 as Time.Micro), timestamp: Time.Timestamp.now() }); + track.writeGroup(group); + await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + + group.close(new NetError.Stream(StreamCode.Cancel)); + writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); + await settle(); + + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); + + consumer.close(); +}); + +// An open group may still have a tail in flight, so adjacency proves nothing when it is retired. +test("Consumer reads a gap when retiring an open group", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 150 as Time.Milli }); + + const group = new Group.Producer(0); + group.writeFrame({ payload: encodeLegacy(0 as Time.Micro), timestamp: Time.Timestamp.now() }); + track.writeGroup(group); + await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + + writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); + await settle(); + + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); + + group.close(); + consumer.close(); +}); + // --- Buffered signal --- test("Consumer buffered signal updates as frames arrive", async () => { diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 1351bdfe1c0..a4bf7c3c188 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -366,8 +366,12 @@ export class Consumer { // Unread media is thrown away, and `first.end` counts it, so contiguity alone would hide it. // A trailing endpoint carries no media, so leaving it unread discards nothing. const discarded = first.frames.some((frame) => this.#format.end?.(frame) === undefined); - const continued = continues(first, next) || ptsContiguous(first.end ?? this.#presentedEnd, nextStart); - if (marker || discarded || !continued) this.#lost = true; + // Adjacency proves nothing is missing only once the group finished cleanly: a truncated + // or still-open group may have lost its tail. + const finished = first.done && !first.truncated; + const adjacent = next?.consumer.sequence === first.consumer.sequence + 1; + const continued = (finished && adjacent) || ptsContiguous(first.end ?? this.#presentedEnd, nextStart); + if (marker || discarded || first.truncated || !continued) this.#lost = true; first.consumer.close(); first.frames.length = 0; skipped++; diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 51102c62051..a83066c24af 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -239,11 +239,12 @@ impl Consumer { // first arrivals instead of assuming group 0. Skipping stays the budget's job. if self.startup { // NOTE: poll_min_timestamp buffers at least one frame per group and - // registers the waiter on the ones still empty. - let any_frame = self - .pending - .iter_mut() - .any(|group| matches!(group.poll_min_timestamp(waiter, &self.format), Poll::Ready(Ok(_)))); + // registers the waiter on the ones still empty. Any wire frame counts, so a + // paused track whose only group is a marker still reads its end. + let any_frame = self.pending.iter_mut().any(|group| { + let _ = group.poll_min_timestamp(waiter, &self.format); + !group.empty + }); if any_frame { self.current = self.pending.front().expect("a group has a frame").sequence; self.startup = false; @@ -963,6 +964,26 @@ mod tests { items } + /// Joining a paused track, whose only served group is a marker, reads the end without + /// waiting for media to resume. + #[tokio::test(start_paused = true)] + async fn a_join_on_a_marker_group_reads_its_end() { + let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscription = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); + write_marker_group(&mut track, 16, ts(1_000_000)); + + let mut read = async || { + tokio::time::timeout(Duration::from_secs(1), reader.read()) + .await + .expect("the end is read while the track stays open") + .unwrap() + }; + assert!(matches!(read().await, Some(Read::Gap))); + assert!(matches!(read().await, Some(Read::End { group: 16, end }) if end == ts(1_000_000))); + } + #[tokio::test] async fn gaps_mark_the_join_and_holes_while_a_marker_reads_as_an_end() { let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); From 19dc728425f1a2e8bc0cbc53b683f947035d375b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 13 Oct 2026 09:24:46 -0700 Subject: [PATCH 10/10] fix(cli): a declared end lets play drain the pictures before it A marker group's end restarted video playback the moment it was read, discarding encoded pictures still buffered before it. Restart at the first frame from a later group instead, as on main. Co-Authored-By: Claude Opus 5.5 --- rs/moq-cli/src/play/video.rs | 58 ++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 890b1c628e3..35a0cc608e5 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -73,12 +73,19 @@ impl Video { pub async fn run(self, mut track: Track, mut decoder: impl Decoder) -> anyhow::Result<()> { let buffer = Mutex::new(Buffer::default()); let receive = async { + // The group of a declared end whose restart waits for a later group, so the + // media buffered before the end still plays out. + let mut end = None; loop { let read = track.read().await?; let mut buffer = buffer.lock().unwrap(); let before = buffer.generation; let ended = match read { - Some(Read::Frame { frame, .. }) => { + Some(Read::Frame { group, frame }) => { + if end.take_if(|end| *end != group).is_some() { + buffer.clear(); + self.presentation.lock().unwrap().video_restarted(); + } if self .presentation .lock() @@ -91,11 +98,16 @@ impl Video { false } // Applied in delivery order, so the restart lands before the frame after it. - Some(Read::Gap | Read::End { .. }) => { + Some(Read::Gap) => { + end = None; buffer.clear(); self.presentation.lock().unwrap().video_restarted(); false } + Some(Read::End { group, .. }) => { + end = Some(group); + false + } None => { buffer.ended = true; true @@ -486,6 +498,48 @@ mod tests { assert_eq!(media.output.present(&media).unwrap().as_millis(), 500); } + #[tokio::test] + async fn a_declared_end_plays_out_the_pictures_before_it() { + tokio::time::pause(); + let delay = Duration::from_secs(2); + let media = media(delay); + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = track + .consume() + .subscribe(moq_net::track::Subscription::default().with_max_delay(delay)) + .await + .unwrap(); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + ); + let track = moq_mux::container::Consumer::new( + subscriber, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video), + ); + for ms in [0, 33, 66] { + producer.write(frame(ms, ms == 0)).unwrap(); + } + // The publisher pauses and sends nothing more for now. + producer.discontinuity().unwrap(); + // A codec that holds nothing back, so every picture is owed before the track ends. + let task = tokio::spawn(playback(&media, delay).run(track, Buffered::new(0))); + let mut shown = Vec::new(); + for _ in 0..2_200 { + tokio::task::yield_now().await; + if let Some(timestamp) = media.output.present(&media) { + shown.push(timestamp.as_millis()); + } + tokio::time::advance(Duration::from_millis(1)).await; + } + assert_eq!(shown, [0, 33, 66], "the end discarded the pictures before it"); + producer.finish().unwrap(); + task.await.unwrap().unwrap(); + } + /// Play one burst of 33 ms pictures, given by slot in decode order, through a /// codec holding `depth` pictures, and check the window shows every one on /// time.