From 49c7a1e19761cee9ef3d6cfe5911c223104bdf06 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 14:04:26 -0700 Subject: [PATCH 01/19] chore(quest): claim Rust frame continuity signaling Co-Authored-By: GPT-6 From f0dc21e28c3c683f8774c35d69d240f26d65faf7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 15:14:44 -0700 Subject: [PATCH 02/19] feat(mux): expose container and audio delivery continuity Co-Authored-By: GPT-6 --- doc/lib/rs/moq-mux.md | 4 + quest/m1/rust-continuous.md | 15 +++ rs/hang/examples/subscribe.rs | 6 +- rs/moq-audio/src/decode/consumer.rs | 135 ++++++++++++++----- rs/moq-audio/src/encode/capture.rs | 2 +- rs/moq-audio/src/encode/producer.rs | 29 ++-- rs/moq-audio/src/playback.rs | 2 +- rs/moq-audio/tests/import.rs | 2 +- rs/moq-audio/tests/roundtrip.rs | 7 +- rs/moq-c/src/audio.rs | 2 +- rs/moq-c/src/consume.rs | 2 +- rs/moq-cli/src/play/media.rs | 2 +- rs/moq-cli/src/play/video.rs | 3 +- rs/moq-cli/src/publish.rs | 3 +- rs/moq-ffi/src/audio.rs | 2 +- rs/moq-ffi/src/media.rs | 6 +- rs/moq-gst/src/source/imp.rs | 2 +- rs/moq-mux/src/codec/opus/import.rs | 4 +- rs/moq-mux/src/codec/vp8/import.rs | 4 +- rs/moq-mux/src/container/consumer.rs | 140 +++++++++++++------- rs/moq-mux/src/container/flv/import_test.rs | 20 ++- rs/moq-mux/src/container/group.rs | 6 +- rs/moq-mux/src/container/mkv/import_test.rs | 2 +- rs/moq-mux/src/container/mod.rs | 9 ++ rs/moq-mux/src/container/producer.rs | 2 +- rs/moq-mux/src/container/test_util.rs | 2 +- rs/moq-mux/src/container/ts/export_test.rs | 8 +- rs/moq-mux/src/container/ts/import.rs | 24 ++-- rs/moq-mux/src/container/ts/import_test.rs | 13 +- rs/moq-mux/src/container/ts/programs.rs | 7 +- rs/moq-mux/src/import/track.rs | 10 +- rs/moq-rtc/src/codec/mod.rs | 8 +- rs/moq-srt/src/ts.rs | 9 +- rs/moq-video/src/capture/wgc.rs | 2 +- rs/moq-video/src/decode/consumer.rs | 6 +- rs/moq-video/src/decode/decoder.rs | 4 +- rs/moq-video/src/encode/producer.rs | 2 +- 37 files changed, 341 insertions(+), 165 deletions(-) diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index 814518cff1e..e90fba94072 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -30,6 +30,10 @@ set at the init segment, so a new rendition or a changed configuration ends the export. The [CLI page](/bin/cli#import) covers these and the MPEG-TS specifics: one program per broadcast, damaged packets, and feed checks. +Container and audio reads return a frame with its delivery continuity: the first +frame and the first after a skip, hole, or declared break start a new span. +Video decoders return bare frames. + Data tracks are catalog entries too. The [hang page](/concept/hang#data-tracks) describes the modes. A capture time converts onto the broadcast clock; a timestamp taken from the source is published as given and lines up with media diff --git a/quest/m1/rust-continuous.md b/quest/m1/rust-continuous.md index a6032b7b82d..c1448c53401 100644 --- a/quest/m1/rust-continuous.md +++ b/quest/m1/rust-continuous.md @@ -28,6 +28,21 @@ continuous. Public API: breaking in moq-mux (the consumer's read result changes). Wire: none. +### Draft progress and remaining video propagation + +The draft adds a shared Sample envelope to container and audio reads, with +regressions for subscribe, markers, holes, skips, and audio decode. Existing +consumers explicitly unwrap the frame; binding surfaces stay unchanged. + +Video still returns its existing decoded Frame. A decoder can return a picture +from before a marker while processing the access unit after it, or return it +only on flush. A single pending boolean attaches the new boundary to the old +picture and loses the new picture boundary, so that approach is not included. +Carry input continuity through codec output association before extending the +video read result. Prefer private decoder metadata association over a +timestamp heuristic, and cover delayed output on both sides of a marker. +The video propagation goal remains open here. + ## Related - [Consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md) - withholds frames after the same signal diff --git a/rs/hang/examples/subscribe.rs b/rs/hang/examples/subscribe.rs index b929f4b8c7e..9622449b2e0 100644 --- a/rs/hang/examples/subscribe.rs +++ b/rs/hang/examples/subscribe.rs @@ -117,9 +117,9 @@ async fn play(consumer: moq_net::origin::Consumer, announce: moq_net::announce:: // Read frames in latency-bounded presentation order. while let Some(frame) = ordered.read().await? { tracing::info!( - timestamp = ?frame.timestamp, - keyframe = frame.keyframe, - bytes = frame.payload.len(), + timestamp = ?frame.frame.timestamp, + keyframe = frame.frame.keyframe, + bytes = frame.frame.payload.len(), "received frame" ); } diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 8875b579724..70f444e4755 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -1,5 +1,6 @@ //! Subscribe to an encoded audio track and emit raw PCM. +use moq_mux::container::Sample; use std::collections::VecDeque; use std::time::{Duration, Instant}; @@ -84,7 +85,7 @@ pub struct Consumer { next_start: Option, /// Frames decoded and not yet handed back, so a gap's tail can be returned /// ahead of the packet that exposed it. - ready: VecDeque, + ready: VecDeque>, /// Codec activity spans the resampler's buffered output still covers. spans: VecDeque, /// Activity of the last span the output ran past, for the rounding samples the @@ -103,6 +104,7 @@ pub struct Consumer { terminal_start: Option, /// Last container playhead generation applied to timeline state. discontinuity: u64, + continuous: bool, /// The playout delay, fixed or estimated from arrivals. target: Target, /// Where arrival times are measured from. @@ -208,6 +210,7 @@ impl Consumer { end: None, terminal_start: None, discontinuity: 0, + continuous: false, origin: now(), }) } @@ -250,6 +253,7 @@ impl Consumer { } /// Read the next decoded PCM frame, or `None` when the track ends. + /// The returned sample carries the container's delivery continuity through codec buffering. /// /// [`Frame::activity`] reports whether the packet these samples came from /// coded audio. It describes where the frame begins, so a resampled @@ -261,7 +265,7 @@ impl Consumer { /// side stay anchored to their own packet timeline, so the hole is there to /// see. "Doesn't continue" allows for the quantization the stamps carry, which /// on a millisecond-stamped ingest is most of a millisecond. - pub async fn read(&mut self) -> Result, Error> { + pub async fn read(&mut self) -> Result>, Error> { loop { if let Some(frame) = self.ready.pop_front() { return Ok(Some(frame)); @@ -269,10 +273,14 @@ impl Consumer { let mux_frame = self.track.read().await?; self.apply_discontinuity()?; - let Some(mux_frame) = mux_frame else { - return self.flush(); + let Some(sample) = mux_frame else { + return Ok(self.flush()?.map(|frame| { + let continuous = std::mem::replace(&mut self.continuous, true); + Sample { frame, continuous } + })); }; + let mux_frame = sample.frame; // Every packet the container hands over is an arrival, including one the // decoder is about to refuse, so it is observed before anything can fail. let arrival = jitter::millis(now().duration_since(self.origin)); @@ -301,9 +309,11 @@ impl Consumer { .is_some_and(|next| discontinuous(next, mux_frame.timestamp)) && let Some(frame) = self.gap()? { - self.ready.push_back(frame); + let continuous = std::mem::replace(&mut self.continuous, true); + self.ready.push_back(Sample { frame, continuous }); } + self.continuous &= sample.continuous; let rate = self.decoder.sample_rate(); let epoch = *self.epoch.get_or_insert(mux_frame.timestamp); let delay = self.decoder.delay_remaining(); @@ -397,7 +407,8 @@ impl Consumer { // Queued rather than returned, so a tail drained at a gap earlier in this // same iteration still comes out first. The next turn of the loop pops it. let frame = self.frame(pcm, timestamp, activity)?; - self.ready.push_back(frame); + let continuous = std::mem::replace(&mut self.continuous, true); + self.ready.push_back(Sample { frame, continuous }); } } @@ -595,6 +606,51 @@ mod tests { use crate::encode::{Encoder, Input, Options as EncodeOptions, Producer, Settings}; use crate::{Format, Layout}; + #[tokio::test] + async fn continuity_survives_audio_decode_and_a_marker() { + use crate::decode::decoder::tests::{opus_catalog, opus_packets}; + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let subscriber = broadcast.consume(); + let catalog = opus_catalog(moq_mux::codec::opus::Config::new(48_000, 1).with_pre_skip(312)); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + let mut consumer = Consumer::new( + &subscriber, + &catalog, + "audio", + Options { + max_delay: std::time::Duration::from_secs(10), + ..Options::new() + }, + ) + .await + .unwrap(); + for (index, payload) in opus_packets(4).into_iter().enumerate() { + if index == 2 { + producer.discontinuity().unwrap(); + } + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_micros(index as u64 * 20_000).unwrap(), + duration: None, + payload, + keyframe: index % 2 == 0, + }) + .unwrap(); + } + producer.finish().unwrap(); + let mut flags = Vec::new(); + while let Some(sample) = consumer.read().await.unwrap() { + flags.push(sample.continuous); + } + assert_eq!(flags, [false, true, false, true]); + } + #[tokio::test] async fn remixes_mono_stream_to_stereo_output() { let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -637,7 +693,7 @@ mod tests { producer.write(&Frame::new(data.into(), Timestamp::ZERO)).unwrap(); let frame = consumer.read().await.unwrap().expect("decoded frame"); - let samples = Format::F32.as_interleaved_f32(&frame.data, 2).unwrap(); + let samples = Format::F32.as_interleaved_f32(&frame.frame.data, 2).unwrap(); assert_eq!(samples.len(), (960 - 312) * 2); for pair in samples.as_chunks::<2>().0.iter() { assert_eq!(pair[0], pair[1]); @@ -681,8 +737,8 @@ mod tests { // 312 samples at 48 kHz is 6.5 ms. for (micros, frames) in [(0, 960 - 312), (13_500, 960), (33_500, 960)] { let frame = consumer.read().await.unwrap().expect("decoded frame"); - assert_eq!(frame.timestamp.as_micros(), micros); - assert_eq!(frame.data.len() / size_of::(), frames); + assert_eq!(frame.frame.timestamp.as_micros(), micros); + assert_eq!(frame.frame.data.len() / size_of::(), frames); } } @@ -737,7 +793,7 @@ mod tests { } let first = consumer.read().await.unwrap().expect("decoded frame"); - assert_eq!(first.timestamp.as_micros(), 0); + assert_eq!(first.frame.timestamp.as_micros(), 0); // Continuity, not a fixed number: the second frame starts where the first // one's samples end, whatever they came to. Within a few frames rather than @@ -745,9 +801,9 @@ mod tests { // wobbles around the nominal ratio; a real hole (the samples it held back, or // the startup silence it dropped) is twenty times this tolerance. let second = consumer.read().await.unwrap().expect("decoded frame"); - let first_frames = (first.data.len() / size_of::()) as u128; + let first_frames = (first.frame.data.len() / size_of::()) as u128; let ends_at = first_frames * 1_000_000 / 48_000; - let gap = second.timestamp.as_micros().abs_diff(ends_at); + let gap = second.frame.timestamp.as_micros().abs_diff(ends_at); assert!(gap < 100, "expected the frames to meet, got a {gap} us gap"); } @@ -797,10 +853,10 @@ mod tests { producer.finish().unwrap(); let first = consumer.read().await.unwrap().expect("decoded frame"); - let first_frames = first.data.len() / size_of::(); + let first_frames = first.frame.data.len() / size_of::(); let tail = consumer.read().await.unwrap().expect("flushed tail"); - let tail_frames = tail.data.len() / size_of::(); + let tail_frames = tail.frame.data.len() / size_of::(); // The 142 held-back frames at 44.1 kHz are ~155 at 48 kHz, plus the 69 the // sinc filter still owes: it runs centred, so the end of the track only @@ -809,7 +865,7 @@ mod tests { // It picks up where the first frame's samples ended, within the same few // frames of whole-frame rounding as above. let ends_at = (first_frames as u128) * 1_000_000 / 48_000; - let gap = tail.timestamp.as_micros().abs_diff(ends_at); + let gap = tail.frame.timestamp.as_micros().abs_diff(ends_at); assert!(gap < 100, "expected the tail to meet the body, got a {gap} us gap"); // Together they cover the packet and no more: 1024 frames at 44.1 kHz is @@ -884,9 +940,9 @@ mod tests { // nothing every other packet. Those must not surface as frames: a frame // with no samples reads as audio arriving, and carries an activity // describing samples that are not there. - assert!(!frame.data.is_empty(), "read returned a frame with no samples"); - if frame.activity.is_dtx() { - actual = Some(frame.timestamp); + assert!(!frame.frame.data.is_empty(), "read returned a frame with no samples"); + if frame.frame.activity.is_dtx() { + actual = Some(frame.frame.timestamp); break; } } @@ -954,7 +1010,10 @@ mod tests { let mut read = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - read.push((frame.timestamp.as_micros(), frame.data.len() / size_of::())); + read.push(( + frame.frame.timestamp.as_micros(), + frame.frame.data.len() / size_of::(), + )); } read } @@ -1105,9 +1164,9 @@ mod tests { // The output begins with the first packet's samples, so it is stamped and // labelled from that packet. Rewinding from the terminal one instead drops it // most of a second into the future, carrying the DTX label with it. - assert_eq!(frame.timestamp.as_micros(), 0, "held samples moved with the jump"); + assert_eq!(frame.frame.timestamp.as_micros(), 0, "held samples moved with the jump"); assert!( - frame.activity.is_active(), + frame.frame.activity.is_active(), "held samples took the terminal packet's label" ); } @@ -1205,20 +1264,24 @@ mod tests { // shortfall is codec delay, not a hole: without counting it the packet after // every stream start would read as a gap. let first = consumer.read().await.unwrap().expect("decoded frame"); - let frames = first.data.len() / size_of::(); + let frames = first.frame.data.len() / size_of::(); assert!(frames < 960, "the pre-skip should be trimmed, got {frames} frames"); // The hole is real, so codec prediction starts over but stream-level pre-skip // does not. The audio is stamped where the packet says rather than 2.5 ms early. let second = consumer.read().await.unwrap().expect("decoded frame"); - assert_eq!(second.timestamp.as_micros(), 22_500); - assert_eq!(second.data.len() / size_of::(), 960, "pre-skip was reapplied"); + assert_eq!(second.frame.timestamp.as_micros(), 22_500); + assert_eq!( + second.frame.data.len() / size_of::(), + 960, + "pre-skip was reapplied" + ); let third = consumer.read().await.unwrap().expect("decoded frame after gap"); - let second_frames = second.data.len() / size_of::(); + let second_frames = second.frame.data.len() / size_of::(); assert_eq!( - third.timestamp, - advance(second.timestamp, second_frames, 48_000).unwrap() + third.frame.timestamp, + advance(second.frame.timestamp, second_frames, 48_000).unwrap() ); } @@ -1351,7 +1414,7 @@ mod tests { let frame = consumer.read().await.unwrap().expect("decoded frame"); assert!( - frame.data.len() / size_of::() < encoder.frame_size(), + frame.frame.data.len() / size_of::() < encoder.frame_size(), "pre-skip is trimmed" ); // The cold-start prior reads 100 ms, plus one 20 ms packet. @@ -1456,9 +1519,9 @@ mod tests { let first = consumer.read().await.unwrap().expect("first decoded frame"); let second = consumer.read().await.unwrap().expect("second decoded frame"); - let first_frames = first.data.len() / size_of::(); - let expected = advance(first.timestamp, first_frames, 48_000).unwrap(); - assert_eq!(second.timestamp, expected); + let first_frames = first.frame.data.len() / size_of::(); + let expected = advance(first.frame.timestamp, first_frames, 48_000).unwrap(); + assert_eq!(second.frame.timestamp, expected); } #[tokio::test] @@ -1514,8 +1577,8 @@ mod tests { let first = consumer.read().await.unwrap().expect("first decoded frame"); let _second = consumer.read().await.unwrap().expect("second decoded frame"); let resumed = consumer.read().await.unwrap().expect("resumed decoded frame"); - let first_frames = first.data.len() / size_of::(); - let resumed_frames = resumed.data.len() / size_of::(); + let first_frames = first.frame.data.len() / size_of::(); + let resumed_frames = resumed.frame.data.len() / size_of::(); assert!(first_frames < frame_size, "the first epoch trims pre-skip"); assert_eq!( resumed_frames, first_frames, @@ -1566,7 +1629,7 @@ mod tests { let frame = consumer.read().await.unwrap().expect("decoded frame"); assert_eq!( - Format::F32.as_interleaved_f32(&frame.data, 1).unwrap().as_ref(), + Format::F32.as_interleaved_f32(&frame.frame.data, 1).unwrap().as_ref(), samples ); } @@ -1622,8 +1685,8 @@ mod tests { let frame = consumer.read().await.unwrap().expect("decoded frame"); // `as_micros`, not `==`: the CMAF path carries the fmp4 timescale and // `Timestamp`'s equality is structural, so the scales would have to match too. - assert_eq!(frame.timestamp.as_micros(), 0); - let samples = Format::F32.as_interleaved_f32(&frame.data, 2).unwrap(); + assert_eq!(frame.frame.timestamp.as_micros(), 0); + let samples = Format::F32.as_interleaved_f32(&frame.frame.data, 2).unwrap(); assert_eq!(samples.len(), (960 - 312) * 2); } } diff --git a/rs/moq-audio/src/encode/capture.rs b/rs/moq-audio/src/encode/capture.rs index 39cff5d6122..8c69e185e64 100644 --- a/rs/moq-audio/src/encode/capture.rs +++ b/rs/moq-audio/src/encode/capture.rs @@ -2194,7 +2194,7 @@ mod tests { async fn read_new(&self, track: &mut Track, seen: &[u64]) -> u64 { loop { let packet = track.read().await.unwrap().expect("a published packet"); - let timestamp = u64::try_from(packet.timestamp.as_micros()).unwrap(); + let timestamp = u64::try_from(packet.frame.timestamp.as_micros()).unwrap(); if !seen.contains(×tamp) { return timestamp; } diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index d92de5745b7..cab2d23f235 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -648,7 +648,7 @@ mod tests { let mut decoded = Vec::new(); while let Some(frame) = audio.read().await.unwrap() { - let pcm = Format::F32.as_interleaved_f32(&frame.data, 1).unwrap(); + let pcm = Format::F32.as_interleaved_f32(&frame.frame.data, 1).unwrap(); decoded.extend_from_slice(&pcm); } assert_eq!(decoded.len(), frames, "terminal padding extended the source"); @@ -819,7 +819,7 @@ mod tests { producer.write(&full_frame(0)).unwrap(); let first = audio.read().await.unwrap().expect("first epoch packet"); - assert_eq!(first.data.len() / size_of::(), 960 - 312); + assert_eq!(first.frame.data.len() / size_of::(), 960 - 312); producer.reset_epoch(); producer.write(&full_frame(1_000_000)).unwrap(); @@ -827,8 +827,8 @@ mod tests { let mut resumed_frames = 0; while let Some(frame) = audio.read().await.unwrap() { - if frame.timestamp.as_micros() >= 1_000_000 { - resumed_frames += frame.data.len() / size_of::(); + if frame.frame.timestamp.as_micros() >= 1_000_000 { + resumed_frames += frame.frame.data.len() / size_of::(); } } assert!(resumed_frames > 0, "the resumed epoch still decodes"); @@ -866,8 +866,8 @@ mod tests { for index in 0..100 { producer.write(&pcm_frame(&silence, index * 20_000)).unwrap(); let consumed = consumer.read().await.unwrap().expect("one decoded frame"); - assert_eq!(producer.activity(), consumed.activity); - if consumed.activity.is_dtx() { + assert_eq!(producer.activity(), consumed.frame.activity); + if consumed.frame.activity.is_dtx() { entered_dtx = true; break; } @@ -883,7 +883,7 @@ mod tests { producer.write(&pcm_frame(&active, 2_000_000)).unwrap(); let consumed = consumer.read().await.unwrap().expect("one decoded frame"); assert_eq!(producer.activity(), Activity::Active); - assert_eq!(consumed.activity, Activity::Active); + assert_eq!(consumed.frame.activity, Activity::Active); } // One 20 ms Opus frame at 48 kHz mono is exactly 960 f32 samples, so each @@ -929,7 +929,7 @@ mod tests { } producer.write(frame).unwrap(); let read = reader.read().await.unwrap().expect("a packet per full frame"); - pts.push(read.timestamp.as_micros()); + pts.push(read.frame.timestamp.as_micros()); } pts } @@ -1074,7 +1074,16 @@ mod tests { producer.write(&pcm_frame(&[0.1; 4 * 1024], epoch_us)).unwrap(); let mut pts = Vec::new(); for _ in 0..4 { - pts.push(reader.read().await.unwrap().expect("a packet").timestamp.as_micros()); + pts.push( + reader + .read() + .await + .unwrap() + .expect("a packet") + .frame + .timestamp + .as_micros(), + ); } pts } @@ -1124,7 +1133,7 @@ mod tests { producer.write(&pcm_frame(&vec![0.1; 44_100], 1_000_000)).unwrap(); let first = reader.read().await.unwrap().expect("a packet"); - assert_eq!(first.timestamp.as_micros(), 1_000_000); + assert_eq!(first.frame.timestamp.as_micros(), 1_000_000); } #[tokio::test] diff --git a/rs/moq-audio/src/playback.rs b/rs/moq-audio/src/playback.rs index 798bb5984dc..884a06a48cf 100644 --- a/rs/moq-audio/src/playback.rs +++ b/rs/moq-audio/src/playback.rs @@ -18,7 +18,7 @@ //! let mut sink = engine.sink(input)?; //! //! while let Some(frame) = audio.read().await? { -//! let write = sink.write(&frame.data)?; +//! let write = sink.write(&frame.frame.data)?; //! if write.dropped_sample_frames > 0 { //! eprintln!("dropped {} live audio frames", write.dropped_sample_frames); //! } diff --git a/rs/moq-audio/tests/import.rs b/rs/moq-audio/tests/import.rs index a1a5a147e43..160f75faeee 100644 --- a/rs/moq-audio/tests/import.rs +++ b/rs/moq-audio/tests/import.rs @@ -39,7 +39,7 @@ async fn ts_surround_opus_decodes() { .expect("decoded frame timed out") .unwrap() { - let (samples, rest) = frame.data.as_chunks::<4>(); + let (samples, rest) = frame.frame.data.as_chunks::<4>(); assert!(rest.is_empty()); for chunk in samples.as_chunks::<6>().0 { for (channel, sample) in energy.iter_mut().zip(chunk) { diff --git a/rs/moq-audio/tests/roundtrip.rs b/rs/moq-audio/tests/roundtrip.rs index 34ba2750638..af63857020c 100644 --- a/rs/moq-audio/tests/roundtrip.rs +++ b/rs/moq-audio/tests/roundtrip.rs @@ -76,6 +76,7 @@ async fn opus_round_trip_48k_stereo() { .unwrap() { let pcm: Vec = frame + .frame .data .as_chunks::<4>() .0 @@ -151,7 +152,7 @@ async fn opus_round_trip_44100_s16_resampled() { .expect("decoded frame timed out") .unwrap() { - total_bytes += frame.data.len() as u64; + total_bytes += frame.frame.data.len() as u64; } assert!( total_bytes > 10_000, @@ -192,8 +193,9 @@ async fn pcm_round_trip_is_lossless() { producer.finish().unwrap(); let frame = consumer.read().await.unwrap().unwrap(); - assert_eq!(frame.timestamp, Timestamp::from_micros(123_000).unwrap()); + assert_eq!(frame.frame.timestamp, Timestamp::from_micros(123_000).unwrap()); let decoded: Vec = frame + .frame .data .as_chunks::<4>() .0 @@ -259,6 +261,7 @@ async fn pcm_five_one_reads_back_at_any_layout() { assert_eq!(consumer.layout(), layout); let decoded = consumer.read().await.unwrap().unwrap(); let samples: Vec = decoded + .frame .data .as_chunks::<4>() .0 diff --git a/rs/moq-c/src/audio.rs b/rs/moq-c/src/audio.rs index f13b3991ba0..5bd108953ba 100644 --- a/rs/moq-c/src/audio.rs +++ b/rs/moq-c/src/audio.rs @@ -272,7 +272,7 @@ impl Audio { }; // Hold the lock only to buffer the frame; release it before the callback. - let frame_id = State::lock().audio.frames.insert(frame)?; + let frame_id = State::lock().audio.frames.insert(frame.frame)?; callback.call(Ok(frame_id)); } } diff --git a/rs/moq-c/src/consume.rs b/rs/moq-c/src/consume.rs index aed2e64588e..114c0aaae14 100644 --- a/rs/moq-c/src/consume.rs +++ b/rs/moq-c/src/consume.rs @@ -540,7 +540,7 @@ impl Consume { }; // Hold the lock only to buffer the frame; release it before the callback. - let frame_id = State::lock().consume.frame.insert(frame)?; + let frame_id = State::lock().consume.frame.insert(frame.frame)?; callback.call(Ok(frame_id)); } } diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index e66f1d88cd4..3e6f3fb9eb4 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -407,7 +407,7 @@ async fn play_audio( loop { let frame = match consumer.read().await { - Ok(Some(frame)) => frame, + Ok(Some(sample)) => sample.frame, Ok(None) => break, // One bad packet is that packet's problem: the decoder stays usable, so // skip it rather than ending playback and taking the video window down diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index e80f837b795..e42366f8c60 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -84,7 +84,8 @@ impl Video { self.presentation.lock().unwrap().video_restarted(); } let ended = frame.is_none(); - if let Some(frame) = frame { + if let Some(sample) = frame { + let frame = sample.frame; if self .presentation .lock() diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 446684f6806..f732db988d8 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -1133,6 +1133,7 @@ pub(crate) mod tests { .await .unwrap() .expect("a video frame") + .frame .timestamp; assert!( first.as_micros() < Duration::from_secs(5).as_micros(), @@ -1294,6 +1295,6 @@ pub(crate) mod tests { .expect("verbatim read timed out") .unwrap() .expect("a published verbatim frame"); - frame.payload.to_vec() + frame.frame.payload.to_vec() } } diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index 7ae6156f7c3..a67218d52cb 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -342,7 +342,7 @@ struct ConsumerInner { impl ConsumerInner { async fn next(&mut self) -> Result, MoqError> { - Ok(self.consumer.read().await?.map(Into::into)) + Ok(self.consumer.read().await?.map(|sample| sample.frame.into())) } } diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 5a51313c1c6..bf58007dbda 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -1096,7 +1096,11 @@ struct Media { impl Media { async fn next(&mut self) -> Result, MoqError> { - self.inner.read().await?.map(media_frame).transpose() + self.inner + .read() + .await? + .map(|sample| media_frame(sample.frame)) + .transpose() } } diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index e0f51009101..8aaef625710 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1080,7 +1080,7 @@ impl Pump { ); gst::event::Segment::new(&segment) }); - let buffer = build_buffer(frame, &mut reference_ts, kind); + let buffer = build_buffer(frame.frame, &mut reference_ts, kind); // pad.push() blocks until downstream accepts the buffer (full queues, a // clock-synced sink). block_in_place hands our sibling tasks to another // worker so a stalled downstream can't pin a runtime thread and starve diff --git a/rs/moq-mux/src/codec/opus/import.rs b/rs/moq-mux/src/codec/opus/import.rs index 3ee70a185a7..ed290b67831 100644 --- a/rs/moq-mux/src/codec/opus/import.rs +++ b/rs/moq-mux/src/codec/opus/import.rs @@ -178,8 +178,8 @@ mod tests { .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.payload.as_ref(), payload); - assert_eq!(frame.timestamp, Timestamp::from_micros(1_000).unwrap()); + assert_eq!(frame.frame.payload.as_ref(), payload); + assert_eq!(frame.frame.timestamp, Timestamp::from_micros(1_000).unwrap()); import.finish().unwrap(); } diff --git a/rs/moq-mux/src/codec/vp8/import.rs b/rs/moq-mux/src/codec/vp8/import.rs index dde79d1f314..f8cb4ce667b 100644 --- a/rs/moq-mux/src/codec/vp8/import.rs +++ b/rs/moq-mux/src/codec/vp8/import.rs @@ -211,8 +211,8 @@ mod tests { .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.payload, keyframe); - assert_eq!(frame.timestamp, Timestamp::from_micros(0).unwrap()); + assert_eq!(frame.frame.payload, keyframe); + assert_eq!(frame.frame.timestamp, Timestamp::from_micros(0).unwrap()); import.finish().unwrap(); } diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index bf6f4fe2d78..4ab0c6b922b 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -3,7 +3,7 @@ use std::task::{Poll, ready}; use moq_net::Timestamp; -use super::{Container, Frame, TimestampRewind}; +use super::{Container, Frame, Sample, TimestampRewind}; /// Media and clean group boundaries in delivery order. pub(crate) enum Event { @@ -94,6 +94,7 @@ pub struct Consumer { // Increments on a declared marker, an unproven delivered hole, and a latency skip. discontinuity: u64, + delivered: Option, // Increments on a declared marker only: the publisher paused or broke its timeline forward. markers: u64, @@ -143,6 +144,7 @@ impl Consumer { floor: None, presented_end: None, discontinuity: 0, + delivered: None, markers: 0, end: None, } @@ -177,7 +179,9 @@ impl Consumer { /// configured max delay. /// /// Returns `None` when the track has ended. - pub async fn read(&mut self) -> Result, F::Error> { + /// The first sample and the first one after a playhead event have `continuous: false`. + /// An undeclared forward timestamp jump alone does not change the flag. + pub async fn read(&mut self) -> Result, F::Error> { kio::wait(|waiter| self.poll_read(waiter)).await } @@ -185,10 +189,14 @@ impl Consumer { /// /// Uses a single waiter that gets registered on all relevant kio channels, /// avoiding the need for `tokio::select!` or `FuturesUnordered`. - pub fn poll_read(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { + pub fn poll_read(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { loop { match ready!(self.poll_event(waiter))? { - Some(Event::Frame(frame)) => return Poll::Ready(Ok(Some(frame))), + Some(Event::Frame(frame)) => { + let continuous = self.delivered == Some(self.discontinuity); + self.delivered = Some(self.discontinuity); + return Poll::Ready(Ok(Some(Sample { frame, continuous }))); + } Some(Event::FrameEnd(end)) => { if self.format.kind() == super::Kind::Audio { self.end = Some(end); @@ -822,7 +830,7 @@ mod tests { let mut frames = Vec::new(); loop { match tokio::time::timeout(Duration::from_millis(200), consumer.read()).await { - Ok(Ok(Some(frame))) => frames.push(frame), + Ok(Ok(Some(frame))) => frames.push(frame.frame), Ok(Ok(None)) => break, Ok(Err(e)) => return Err(e), Err(_) => panic!( @@ -903,6 +911,38 @@ mod tests { group.finish().unwrap(); } + #[tokio::test] + async fn continuity_marks_subscribe_holes_and_markers() { + let mut track = track_producer("continuity", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscription = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); + write_group(&mut track, 0, &[ts(0), ts(10_000)]); + write_group(&mut track, 2, &[ts(1_000_000), ts(1_010_000)]); + write_marker_group(&mut track, 3, ts(1_010_000)); + write_group(&mut track, 4, &[ts(2_000_000), ts(2_010_000)]); + track.finish().unwrap(); + let mut flags = Vec::new(); + while let Some(sample) = reader.read().await.unwrap() { + flags.push(sample.continuous); + } + assert_eq!(flags, [false, true, false, true, false, true]); + } + + #[tokio::test] + async fn continuity_marks_a_latency_skip() { + let mut track = track_producer("continuity", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscription = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); + reader.max_delay = Duration::ZERO; + write_group(&mut track, 0, &[ts(0)]); + write_group(&mut track, 3, &[ts(1_000_000)]); + track.finish().unwrap(); + assert!(!reader.read().await.unwrap().unwrap().continuous); + assert!(!reader.read().await.unwrap().unwrap().continuous); + } + #[tokio::test] async fn empty_group_declares_a_discontinuity() { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); @@ -915,9 +955,9 @@ mod tests { write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1); } @@ -937,11 +977,11 @@ mod tests { write_group(&mut track, 4, &[ts(2_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1, "the hole moves the playhead"); assert_eq!(consumer.markers(), 0, "but the timeline carried on"); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(2_000_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(2_000_000)); assert_eq!(consumer.discontinuity(), 2); assert_eq!(consumer.markers(), 1, "the marker declares a break"); } @@ -962,8 +1002,8 @@ mod tests { write_group(&mut track, 2, &[ts(1_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000)); assert_eq!(consumer.discontinuity(), 0, "an empty group is not a marker"); } @@ -980,9 +1020,9 @@ mod tests { write_group(&mut track, 3, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); assert_eq!( consumer.discontinuity(), 1, @@ -1172,11 +1212,11 @@ mod tests { .unwrap(); }; let read = |consumer: &mut Consumer| match consumer.poll_read(&waiter) { - Poll::Ready(Ok(Some(frame))) => Some(frame.timestamp), + Poll::Ready(Ok(Some(frame))) => Some(frame.frame.timestamp), Poll::Pending => None, other => panic!( "unexpected read: {:?}", - other.map(|r| r.map(|f| f.map(|f| f.timestamp))) + other.map(|r| r.map(|f| f.map(|f| f.frame.timestamp))) ), }; @@ -1214,7 +1254,7 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(100_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); write_group(&mut track, 1, &[ts(0)]); track.finish().unwrap(); let err = consumer.read().await.unwrap_err(); @@ -1248,10 +1288,10 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(100_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); write_group(&mut track, 1, &[ts(100_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(100_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); } /// A keyframe one frame below the previous group's last frame, but above its start, is an @@ -1264,7 +1304,7 @@ mod tests { // Read the first group before the second arrives: a blocked group that the next one // already covers is skipped for latency, which is not what this checks. for expected in [0, 33_000, 66_000] { - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(expected)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(expected)); } write_group(&mut track, 1, &[ts(50_000), ts(83_000)]); track.finish().unwrap(); @@ -1301,9 +1341,9 @@ mod tests { write_marker_group(&mut track, 1, ts(0)); write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1); } @@ -1313,11 +1353,11 @@ mod tests { let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Audio)); consumer.max_delay = Duration::ZERO; write_group(&mut track, 0, &[ts(0)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); write_marker_group(&mut track, 1, ts(0)); write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(1_000_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); assert_eq!( consumer.discontinuity(), 1, @@ -1343,12 +1383,12 @@ mod tests { }], ) .unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); write_marker_group(&mut track, 1, ts(10_000)); write_group(&mut track, 2, &[ts(10_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(10_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(10_000)); assert_eq!( consumer.discontinuity(), 1, @@ -1362,10 +1402,10 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(100_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); write_group(&mut track, 1, &[ts(200_000), ts(50_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(200_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(200_000)); let err = consumer.read().await.unwrap_err(); assert!(matches!( err, @@ -1451,7 +1491,7 @@ mod tests { group.finish().unwrap(); track.finish().unwrap(); - let frame = consumer.read().await.unwrap().unwrap(); + let frame = consumer.read().await.unwrap().unwrap().frame; assert!(frame.keyframe, "the marker does not consume the first-media slot"); assert!(frame.duration.is_none(), "a leading marker has no previous frame"); } @@ -1638,7 +1678,7 @@ mod tests { let result = tokio::time::timeout(Duration::from_millis(500), async { let mut frames = Vec::new(); while let Ok(Some(frame)) = consumer.read().await { - frames.push(frame); + frames.push(frame.frame); } frames }) @@ -1749,7 +1789,7 @@ mod tests { route.requested_broadcast().await.unwrap().accept(&source); track.demand().used().await.unwrap(); if sequence == 1 { - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); } for sequence in 1..=sequence { write_group(&mut track, sequence, &[ts(sequence * 1_000_000)]); @@ -1777,7 +1817,7 @@ mod tests { matches!(consumer.poll_event(&waiter), Poll::Ready(Ok(Some(Event::GroupEnd)))), "the complete successor still emits its clean boundary" ); - assert_eq!(consumer.read().await.unwrap().unwrap().timestamp, ts(2_000_000)); + assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(2_000_000)); } // ---- Eviction recovery (pause/resume) ---- @@ -1807,7 +1847,7 @@ mod tests { }], ) .unwrap(); - let first = consumer.read().await.unwrap().unwrap(); + let first = consumer.read().await.unwrap().unwrap().frame; assert_eq!(first.timestamp, ts(0)); // A live group arrives far ahead -- sequences 1..4 never come (evicted). The @@ -1824,7 +1864,11 @@ mod tests { .expect("consumer hung on an evicted group / gap") .unwrap() .unwrap(); - assert_eq!(next.timestamp, ts(150_000), "skipped the evicted gap to the live group"); + assert_eq!( + next.frame.timestamp, + ts(150_000), + "skipped the evicted gap to the live group" + ); } /// A missing (evicted) sequence with a newer group buffered must be skipped once the @@ -1850,7 +1894,7 @@ mod tests { // Reading must reach group 2 across the gap instead of waiting forever for 1. let reached = tokio::time::timeout(Duration::from_secs(1), async { loop { - let frame = consumer.read().await.unwrap().unwrap(); + let frame = consumer.read().await.unwrap().unwrap().frame; if frame.timestamp == ts(200_000) { return; } @@ -1877,7 +1921,7 @@ mod tests { write_group(&mut track, 2, &[ts(200_000)]); let first = consumer.read().await.unwrap().expect("group 0 frame"); - assert_eq!(first.timestamp, ts(0)); + assert_eq!(first.frame.timestamp, ts(0)); // Group 0 is done; group 1's stream is still racing. Within the budget the // consumer waits at the gap rather than skipping to group 2. @@ -2139,7 +2183,7 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } frames }) @@ -2179,7 +2223,7 @@ mod tests { write_group(&mut track, 0, &[ts(0)]); track.finish().unwrap(); - let frame = consumer.read().await.unwrap().unwrap(); + let frame = consumer.read().await.unwrap().unwrap().frame; assert_eq!(frame.timestamp, ts(0)); consumer.set_max_delay(Duration::from_millis(100)); @@ -2223,7 +2267,7 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } frames }) @@ -2284,7 +2328,7 @@ mod tests { let _frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } frames }) @@ -2434,7 +2478,7 @@ mod tests { .expect("an unfloored join must not wait out a gap below the served start") .unwrap() .expect("track still live"); - assert_eq!(frame.timestamp, ts(3_000_000)); + assert_eq!(frame.frame.timestamp, ts(3_000_000)); } #[tokio::test] @@ -2613,7 +2657,7 @@ mod tests { track.finish().unwrap(); // Take the first frame only, leaving the second unread. - let first = consumer.read().await.unwrap().unwrap(); + let first = consumer.read().await.unwrap().unwrap().frame; assert_eq!(first.timestamp, ts(0)); // Expiry aborts the finished group while the reader is still inside it. @@ -2689,7 +2733,7 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } frames }) @@ -2746,7 +2790,7 @@ mod tests { let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } assert_eq!(frames.len(), 3); @@ -2788,7 +2832,7 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } frames }) @@ -2824,7 +2868,7 @@ mod tests { track.finish().unwrap(); let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } assert_eq!(frames.len(), 2); assert_eq!(consumer.discontinuity(), 0); @@ -2863,7 +2907,7 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame); + frames.push(frame.frame); } frames }) @@ -2898,7 +2942,7 @@ mod tests { .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.timestamp, ts(0)); + assert_eq!(frame.frame.timestamp, ts(0)); write_marker(&mut group, ts(15_000)); let event = kio::wait(|waiter| consumer.poll_event(waiter)).await.unwrap(); assert!(matches!(event, Some(Event::FrameEnd(end)) if end == ts(15_000))); diff --git a/rs/moq-mux/src/container/flv/import_test.rs b/rs/moq-mux/src/container/flv/import_test.rs index 4c92efcf755..4619fcbd84c 100644 --- a/rs/moq-mux/src/container/flv/import_test.rs +++ b/rs/moq-mux/src/container/flv/import_test.rs @@ -182,9 +182,12 @@ async fn import_emits_frames() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let frame = decoder.read().await.unwrap().expect("a video frame"); - assert!(frame.keyframe); + assert!(frame.frame.keyframe); // The payload is the length-prefixed NALU, carried through verbatim. - assert_eq!(frame.payload.as_ref(), &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00]); + assert_eq!( + frame.frame.payload.as_ref(), + &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00] + ); } #[tokio::test(start_paused = true)] @@ -212,7 +215,10 @@ async fn public_container_preserves_loc_for_flv() { .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.payload.as_ref(), &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00]); + assert_eq!( + frame.frame.payload.as_ref(), + &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00] + ); } /// Bytes split across two `decode` calls still reassemble into whole tags. @@ -505,8 +511,8 @@ async fn import_enhanced_av1() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let frame = decoder.read().await.unwrap().expect("an AV1 frame"); - assert!(frame.keyframe); - assert_eq!(frame.payload.as_ref(), payload); + assert!(frame.frame.keyframe); + assert_eq!(frame.frame.payload.as_ref(), payload); } /// GStreamer 1.28 `fdkaacenc` output at 48 kHz stereo, remuxed to FLV by ffmpeg 9.0.1, whose @@ -639,7 +645,7 @@ async fn import_reports_negative_pts_and_can_resume() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let frame = decoder.read().await.unwrap().expect("the good frame"); - assert_eq!(frame.timestamp.as_millis(), 10); + assert_eq!(frame.frame.timestamp.as_millis(), 10); } #[tokio::test(start_paused = true)] @@ -679,7 +685,7 @@ async fn import_enhanced_hvc1_applies_composition_time() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let frame = decoder.read().await.unwrap().expect("a video frame"); - assert_eq!(frame.timestamp.as_millis(), 17); + assert_eq!(frame.frame.timestamp.as_millis(), 17); } #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/container/group.rs b/rs/moq-mux/src/container/group.rs index 92c756af347..fc764235f8a 100644 --- a/rs/moq-mux/src/container/group.rs +++ b/rs/moq-mux/src/container/group.rs @@ -140,9 +140,9 @@ mod tests { Hang::new(&config, crate::container::Kind::Data).unwrap(), ); let first = live.read().await.unwrap().unwrap(); - assert!(first.payload.is_empty()); - assert!(first.keyframe); - assert_eq!(live.read().await.unwrap().unwrap().payload, b"data".as_slice()); + assert!(first.frame.payload.is_empty()); + assert!(first.frame.keyframe); + assert_eq!(live.read().await.unwrap().unwrap().frame.payload, b"data".as_slice()); assert!(live.read().await.unwrap().is_none()); let group = consumer.track("data").unwrap().fetch_group(0, None).await.unwrap(); let mut fetched = GroupConsumer::new(group, Hang::new(&config, crate::container::Kind::Data).unwrap()); diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index 370d8b6a11d..9fad3104ce0 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -268,7 +268,7 @@ async fn public_container_preserves_loc_for_mkv() { .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.payload.as_ref(), payload); + assert_eq!(frame.frame.payload.as_ref(), payload); } #[test] diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 0cc44b14c75..2d0bd24bb5d 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -44,6 +44,15 @@ pub enum Kind { Data, } +/// A delivered frame and whether it continues the previous delivery. +#[derive(Clone, Debug)] +pub struct Sample { + /// The encoded or decoded media frame. + pub frame: T, + /// False on the first frame and after a playhead discontinuity. + pub continuous: bool, +} + /// A decoded media frame: timestamp, payload bytes, keyframe flag. /// /// `payload` is the raw codec bitstream that gets handed to the decoder. diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index 65be578fda4..78793f4ba51 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -1683,7 +1683,7 @@ mod tests { crate::container::Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Video)); let mut timestamps = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - timestamps.push(frame.timestamp.as_micros()); + timestamps.push(frame.frame.timestamp.as_micros()); } assert_eq!(timestamps, [1_000_000, 1_040_000, 1_050_000]); } diff --git a/rs/moq-mux/src/container/test_util.rs b/rs/moq-mux/src/container/test_util.rs index f577875ed45..18853cc3bcb 100644 --- a/rs/moq-mux/src/container/test_util.rs +++ b/rs/moq-mux/src/container/test_util.rs @@ -156,7 +156,7 @@ pub(crate) async fn published( .expect("the importer finished its tracks") .unwrap() { - timestamps.push(frame.timestamp.as_micros()); + timestamps.push(frame.frame.timestamp.as_micros()); } out.insert(name, timestamps); } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index d77ca6e7041..4ea573cf296 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -1299,7 +1299,7 @@ async fn export_scte35_roundtrip() { .unwrap() .expect("no SCTE-35 frame after round-trip"); assert_eq!( - frame.payload.as_ref(), + frame.frame.payload.as_ref(), CUE, "SCTE-35 section did not round-trip byte-for-byte" ); @@ -1415,7 +1415,7 @@ async fn export_pes_verbatim_roundtrip() { .unwrap() .expect("no verbatim PES frame after round-trip"); assert_eq!( - frame.payload.as_ref(), + frame.frame.payload.as_ref(), PAYLOAD, "verbatim PES payload round-trips byte-for-byte" ); @@ -1487,7 +1487,7 @@ async fn read_frames(consumer: &moq_net::broadcast::Consumer, name: &str, kind: let mut frames = Vec::new(); while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; - frames.push(frame.payload.to_vec()); + frames.push(frame.frame.payload.to_vec()); } frames } @@ -2068,7 +2068,7 @@ async fn read_cues(consumer: &moq_net::broadcast::Consumer, name: &str) -> Vec<( let mut cues = Vec::new(); while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; - cues.push((frame.payload.to_vec(), frame.timestamp)); + cues.push((frame.frame.payload.to_vec(), frame.frame.timestamp)); } cues } diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 56b0e02534f..63c4e53e4d8 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -3729,7 +3729,7 @@ pub(super) mod test { .unwrap() .expect("a published cue frame"); assert_eq!( - &frame.payload[..], + &frame.frame.payload[..], &CUE[..], "verbatim splice_info_section after upgrade" ); @@ -4393,7 +4393,7 @@ pub(super) mod test { ); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { - frames.push(frame); + frames.push(frame.frame); } frames } @@ -4447,7 +4447,7 @@ pub(super) mod test { ); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { - frames.push(frame); + frames.push(frame.frame); } frames } @@ -5666,12 +5666,12 @@ pub(super) mod test { .unwrap() .expect("a published cue frame"); - assert_eq!(&frame.payload[..], &CUE[..], "verbatim splice_info_section"); - assert_ne!(frame.timestamp, Timestamp::ZERO, "cue must not stamp zero"); + assert_eq!(&frame.frame.payload[..], &CUE[..], "verbatim splice_info_section"); + assert_ne!(frame.frame.timestamp, Timestamp::ZERO, "cue must not stamp zero"); // The legacy container normalizes the wire timestamp to microseconds, so compare the // instant (not the raw scale) against the 90 kHz media clock the cue was stamped with. assert_eq!( - Duration::from(frame.timestamp), + Duration::from(frame.frame.timestamp), Duration::from(clock), "cue stamped with the video media clock" ); @@ -5851,7 +5851,11 @@ pub(super) mod test { .expect("verbatim read timed out") .unwrap() .expect("a published verbatim frame"); - assert_eq!(&frame.payload[..], &payload[..], "verbatim PES payload round-trips"); + assert_eq!( + &frame.frame.payload[..], + &payload[..], + "verbatim PES payload round-trips" + ); } /// A program of only verbatim PIDs reserves no rendition, so the held reservation alone keeps @@ -5941,7 +5945,7 @@ pub(super) mod test { let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { - frames.push(frame); + frames.push(frame.frame); } (frames, reader.discontinuity()) } @@ -6548,8 +6552,8 @@ pub(super) mod test { let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy(kind)); let mut frames = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { - if !frame.payload.is_empty() { - frames.push(frame); + if !frame.frame.payload.is_empty() { + frames.push(frame.frame); } } frames diff --git a/rs/moq-mux/src/container/ts/import_test.rs b/rs/moq-mux/src/container/ts/import_test.rs index 8b3f0662b39..bf8eb27b512 100644 --- a/rs/moq-mux/src/container/ts/import_test.rs +++ b/rs/moq-mux/src/container/ts/import_test.rs @@ -101,8 +101,8 @@ async fn public_container_preserves_loc_for_ts() { .unwrap() .unwrap() .unwrap(); - assert!(frame.keyframe); - assert!(!frame.payload.is_empty()); + assert!(frame.frame.keyframe); + assert!(!frame.frame.payload.is_empty()); } /// The Kyrion capture is H.264 1080i with B-frames (open-GOP, ~5-frame reorder despite only @@ -264,7 +264,7 @@ async fn import_opus_frames() { let mut frames = Vec::new(); while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { let Some(frame) = res.unwrap() else { break }; - frames.push(frame.payload); + frames.push(frame.frame.payload); } assert!(frames.len() > 5, "expected a run of Opus packets, got {}", frames.len()); @@ -545,7 +545,10 @@ async fn survives_midstream_join() { frames.push(frame); } assert_eq!(frames.len(), 1, "expected only the post-join IDR, got {}", frames.len()); - assert!(frames[0].keyframe, "the first surviving frame must be the keyframe"); + assert!( + frames[0].frame.keyframe, + "the first surviving frame must be the keyframe" + ); } /// A real Ateme Kyrion broadcast captured mid-stream with `nc`, so it opens dirty: @@ -596,7 +599,7 @@ async fn kyrion_dirtystart_extracts_real_cues() { ); let mut cues = Vec::new(); while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { - cues.push((frame.payload.to_vec(), frame.timestamp)); + cues.push((frame.frame.payload.to_vec(), frame.frame.timestamp)); } assert_eq!(cues.len(), 6, "expected the six real splice_inserts"); assert!( diff --git a/rs/moq-mux/src/container/ts/programs.rs b/rs/moq-mux/src/container/ts/programs.rs index 291c8c49f00..c6c59b4ad51 100644 --- a/rs/moq-mux/src/container/ts/programs.rs +++ b/rs/moq-mux/src/container/ts/programs.rs @@ -380,12 +380,15 @@ mod test { let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let container = Container::try_from(config).unwrap(); let frame = Consumer::new(track, container).read().await.unwrap().expect("a frame"); - assert!(fills.contains(&frame.payload[4]), "{path} carries only its own program"); + assert!( + fills.contains(&frame.frame.payload[4]), + "{path} carries only its own program" + ); let skew = Duration::from_secs(2); let wall = catalog .clock .expect("the catalog advertises a clock") - .wall_clock(frame.timestamp) + .wall_clock(frame.frame.timestamp) .unwrap(); assert!( before - skew <= wall && wall <= after + skew, diff --git a/rs/moq-mux/src/import/track.rs b/rs/moq-mux/src/import/track.rs index 9aca70feae6..208b58d8fed 100644 --- a/rs/moq-mux/src/import/track.rs +++ b/rs/moq-mux/src/import/track.rs @@ -1055,8 +1055,8 @@ mod tests { .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.payload, payload); - assert_eq!(frame.timestamp, Timestamp::from_micros(1_000).unwrap()); + assert_eq!(frame.frame.payload, payload); + assert_eq!(frame.frame.timestamp, Timestamp::from_micros(1_000).unwrap()); import.finish().unwrap(); } @@ -1383,9 +1383,9 @@ mod tests { .unwrap() .unwrap() .unwrap(); - assert!(frame.keyframe, "an IDR access unit must open a group"); - assert_eq!(frame.payload, au, "the payload must pass through verbatim"); - assert_eq!(frame.timestamp, Timestamp::from_micros(1_000).unwrap()); + assert!(frame.frame.keyframe, "an IDR access unit must open a group"); + assert_eq!(frame.frame.payload, au, "the payload must pass through verbatim"); + assert_eq!(frame.frame.timestamp, Timestamp::from_micros(1_000).unwrap()); import.finish().unwrap(); } diff --git a/rs/moq-rtc/src/codec/mod.rs b/rs/moq-rtc/src/codec/mod.rs index f83ca447faf..5b3a6c4f0ff 100644 --- a/rs/moq-rtc/src/codec/mod.rs +++ b/rs/moq-rtc/src/codec/mod.rs @@ -234,13 +234,13 @@ impl Track { return Ok(None); }; let payload = match &self.convert { - TrackConvert::Passthrough => frame.payload, + TrackConvert::Passthrough => frame.frame.payload, TrackConvert::LengthPrefixed { length_size, keyframe_prefix, } => { - let prefix = frame.keyframe.then(|| keyframe_prefix.as_ref()); - moq_mux::codec::annexb::from_length_prefixed(&frame.payload, *length_size, prefix) + let prefix = frame.frame.keyframe.then(|| keyframe_prefix.as_ref()); + moq_mux::codec::annexb::from_length_prefixed(&frame.frame.payload, *length_size, prefix) .map_err(moq_mux::Error::from)? } }; @@ -248,7 +248,7 @@ impl Track { continue; } return Ok(Some(PacketizedFrame { - timestamp_us: frame.timestamp.as_micros() as u64, + timestamp_us: frame.frame.timestamp.as_micros() as u64, payload, })); } diff --git a/rs/moq-srt/src/ts.rs b/rs/moq-srt/src/ts.rs index b3fdf755bbd..8c376692453 100644 --- a/rs/moq-srt/src/ts.rs +++ b/rs/moq-srt/src/ts.rs @@ -634,7 +634,7 @@ mod tests { .expect("cue read timed out") .unwrap() .expect("a published cue section"); - let expected_cue = cue.payload; + let expected_cue = cue.frame.payload; assert_eq!(expected_cue[0], 0xFC, "a verbatim splice_info_section (table_id 0xFC)"); let mut subscriber = Subscriber::new(&origin.consume(), "ingest", Options::default()) @@ -683,9 +683,12 @@ mod tests { .expect("round-trip cue read timed out") .unwrap() .expect("SRT egress preserves a cue section"); - assert_eq!(cue.payload[0], 0xFC, "the round-trip cue is a splice_info_section"); assert_eq!( - cue.payload, expected_cue, + cue.frame.payload[0], 0xFC, + "the round-trip cue is a splice_info_section" + ); + assert_eq!( + cue.frame.payload, expected_cue, "SRT egress preserves the complete cue section" ); } diff --git a/rs/moq-video/src/capture/wgc.rs b/rs/moq-video/src/capture/wgc.rs index ce0c7ede8c9..d426fc0b6fb 100644 --- a/rs/moq-video/src/capture/wgc.rs +++ b/rs/moq-video/src/capture/wgc.rs @@ -328,7 +328,7 @@ mod tests { let frame = crate::Frame::new(crate::Surface::I420(surface.clone()), timestamp); producer.publish(&encoder.encode(&frame).unwrap()).unwrap(); let received = subscriber.read().await.unwrap().unwrap(); - assert_eq!(received.timestamp.as_micros(), u128::from(step) * 20_000); + assert_eq!(received.frame.timestamp.as_micros(), u128::from(step) * 20_000); } } diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 7704dd365b7..8e4259ad055 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -194,7 +194,11 @@ impl Consumer { self.pending.extend( self.decoder - .decode(mux_frame.payload, mux_frame.timestamp, mux_frame.keyframe) + .decode( + mux_frame.frame.payload, + mux_frame.frame.timestamp, + mux_frame.frame.keyframe, + ) .await?, ); } diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index 732930d15aa..1d2b5a27efc 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -597,13 +597,13 @@ mod tests { let mut track = moq_mux::container::Consumer::new(track, moq_mux::catalog::hang::Container::try_from(config).unwrap()); let sample = track.read().await.unwrap().expect("an imported sample"); - assert!(sample.keyframe); + assert!(sample.frame.keyframe); let decoder = super::Decoder::new(config, &decode_config(super::Kind::Named(probe::BUFFERED_NAME.into()))).unwrap(); let access_unit = decoder .conversion - .access_unit(&sample.payload, sample.keyframe) + .access_unit(&sample.frame.payload, sample.frame.keyframe) .unwrap(); let mut buf = access_unit.clone(); let mut nals = moq_mux::codec::annexb::NalIterator::new(&mut buf); diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index 44be26c103a..de8a7aa3227 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -1209,7 +1209,7 @@ mod tests { async fn read(track: &mut moq_mux::container::Consumer) -> u64 { let frame = track.read().await.unwrap().expect("a published frame"); - u64::try_from(frame.timestamp.as_micros()).unwrap() + u64::try_from(frame.frame.timestamp.as_micros()).unwrap() } /// Read the next frame not already in `seen`: a resubscription replays retained groups first. From bf8973101a7e136804797a2200bd0bc84f38d7e2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 17:30:47 -0700 Subject: [PATCH 03/19] fix(audio): preserve continuity across resampler tails Co-Authored-By: GPT-6 Astra --- quest/m1/rust-continuous.md | 16 ++++- rs/moq-audio/src/decode/consumer.rs | 92 +++++++++++++++++++++++++---- 2 files changed, 95 insertions(+), 13 deletions(-) diff --git a/quest/m1/rust-continuous.md b/quest/m1/rust-continuous.md index c1448c53401..7bb0604a03f 100644 --- a/quest/m1/rust-continuous.md +++ b/quest/m1/rust-continuous.md @@ -41,7 +41,21 @@ picture and loses the new picture boundary, so that approach is not included. Carry input continuity through codec output association before extending the video read result. Prefer private decoder metadata association over a timestamp heuristic, and cover delayed output on both sides of a marker. -The video propagation goal remains open here. +The video propagation goal remains open here. Native backends need explicit +input identity, not a map keyed only by presentation time: equal timestamps +can straddle a marker. VideoToolbox and Media Foundation already map synthetic +sample times, OpenH264 queues presentation times, and VAAPI, NVDEC and +MediaCodec carry codec timestamps. Media Foundation also has an unmatched-output +fallback. Decide how a backend reports an output with no surviving input +identity before exposing continuity: recommend an explicit error, rather than +borrowing the latest input's flag. Keep dropped-input bookkeeping bounded and +cover delayed output, duplicate timestamps and decoder recovery on each backend. + +The audio path now drains a resampler's buffered tail before applying a +playhead boundary. A paused-time regression previously returned flags +`false, false, true`, incorrectly joining the old tail into the new span; +it now returns the old body/tail and new body/tail as +`false, true, false, true`. ## Related diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 70f444e4755..35f9a3ad7f3 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -67,8 +67,8 @@ impl Options { /// /// The mirror of [`encode::Producer`](crate::encode::Producer): output format / /// sample rate / layout are fixed at construction, and -/// [`read`](Self::read) returns [`Frame`]s carrying the codec activity they -/// were decoded from. +/// [`read`](Self::read) returns samples carrying each frame's delivery continuity +/// and codec activity. pub struct Consumer { decoder: Decoder, track: moq_mux::container::Consumer, @@ -274,10 +274,11 @@ impl Consumer { let mux_frame = self.track.read().await?; self.apply_discontinuity()?; let Some(sample) = mux_frame else { - return Ok(self.flush()?.map(|frame| { + if let Some(frame) = self.flush()? { let continuous = std::mem::replace(&mut self.continuous, true); - Sample { frame, continuous } - })); + self.ready.push_back(Sample { frame, continuous }); + } + return Ok(self.ready.pop_front()); }; let mux_frame = sample.frame; @@ -420,6 +421,12 @@ impl Consumer { return Ok(()); } + // Samples already buffered belong to the old span, even when the next + // packet declares a new one. Drain before clearing their timeline labels. + if let Some(frame) = self.drain()? { + let continuous = std::mem::replace(&mut self.continuous, true); + self.ready.push_back(Sample { frame, continuous }); + } self.discontinuity = discontinuity; self.next_start = None; self.spans.clear(); @@ -444,13 +451,7 @@ impl Consumer { fn gap(&mut self) -> Result, Error> { self.decoder.reset_prediction()?; - let mut frame = None; - if let Some(resampler) = self.resampler.as_mut() { - let held = resampler.held_at(); - let skipped = resampler.skipped(); - let pcm = resampler.drain()?; - frame = self.tail(pcm, held, skipped)?; - } + let frame = self.drain()?; self.next_start = None; self.spans.clear(); @@ -460,6 +461,17 @@ impl Consumer { Ok(frame) } + /// Drain one span while keeping the resampler available for the next. + fn drain(&mut self) -> Result, Error> { + let Some(resampler) = self.resampler.as_mut() else { + return Ok(None); + }; + let held = resampler.held_at(); + let skipped = resampler.skipped(); + let pcm = resampler.drain()?; + self.tail(pcm, held, skipped) + } + /// The tail the resampler is still holding when the track ends, once. /// /// Without it the last partial chunk is dropped, which is up to a chunk of @@ -876,6 +888,62 @@ mod tests { assert!(consumer.read().await.unwrap().is_none()); } + #[tokio::test(start_paused = true)] + async fn resampled_continuity_keeps_the_tail_before_a_marker() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let subscriber = broadcast.consume(); + let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 44_100, 1); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + let mut consumer = Consumer::new( + &subscriber, + &catalog, + "audio", + Options { + max_delay: Duration::from_secs(10), + output: Output { + sample_rate: Some(48_000), + ..Output::default() + }, + ..Options::new() + }, + ) + .await + .unwrap(); + for index in 0..2 { + if index != 0 { + producer.discontinuity().unwrap(); + } + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_micros(index * 1_000_000).unwrap(), + duration: None, + payload: vec![0u8; 1024 * size_of::()].into(), + keyframe: true, + }) + .unwrap(); + } + producer.finish().unwrap(); + let mut samples = Vec::new(); + while let Some(sample) = consumer.read().await.unwrap() { + samples.push((sample.frame.timestamp.as_micros(), sample.continuous)); + } + assert_eq!( + samples.iter().map(|(_, continuous)| *continuous).collect::>(), + [false, true, false, true] + ); + assert!( + samples[1].0 < 1_000_000, + "the old resampler tail stays before the marker" + ); + assert_eq!(samples[2].0, 1_000_000, "the new span starts on its source timestamp"); + } + #[tokio::test] async fn resampling_keeps_the_activity_boundary_on_its_source() { let mut encoder = Encoder::new(&Settings { From 435257967530c0d5bd77e97d126de9e870efe331 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 11 Oct 2026 00:17:39 -0700 Subject: [PATCH 04/19] chore(quest): claim watch arrival diagnostics Co-Authored-By: GPT-6 Astra From 482a7367d0298517b14b4695bd5efb1f9b2b2134 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 11 Oct 2026 00:30:06 -0700 Subject: [PATCH 05/19] feat(watch): expose bounded ingress diagnostics Co-Authored-By: GPT-6 Astra --- .github/workflows/nightly.yml | 4 ++++ doc/lib/js/watch.md | 6 ++++- js/hang/src/container/consumer.test.ts | 6 ++++- js/hang/src/container/consumer.ts | 20 ++++++++++++---- js/hang/src/container/types.ts | 6 +++++ js/justfile | 5 ++++ js/watch/bench/arrivals.ts | 27 ++++++++++++++++++++++ js/watch/src/arrivals.test.ts | 30 ++++++++++++++++++++++++ js/watch/src/arrivals.ts | 25 ++++++++++++++++++++ js/watch/src/audio/decoder.test.ts | 14 ++++++++++- js/watch/src/audio/decoder.ts | 13 ++++++++--- js/watch/src/audio/replay.test.ts | 32 +++++++++++++++++++++++++- js/watch/src/audio/replay.ts | 31 ++++++++++++++++++++++--- js/watch/src/index.ts | 1 + js/watch/src/sync.test.ts | 18 +++++++++++++++ js/watch/src/sync.ts | 13 ++++++----- js/watch/src/video/decoder.test.ts | 10 ++++++-- js/watch/src/video/decoder.ts | 17 ++++++++++---- 18 files changed, 251 insertions(+), 27 deletions(-) create mode 100644 js/watch/bench/arrivals.ts create mode 100644 js/watch/src/arrivals.test.ts create mode 100644 js/watch/src/arrivals.ts diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 846119999f4..0ee35540268 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -137,6 +137,10 @@ jobs: if: ${{ !cancelled() }} run: nix develop --command just js catalog-bench + - name: JS arrival window benchmark + if: ${{ !cancelled() }} + run: nix develop --command just js arrivals-bench + # Times Effect create and close beside a bare `new Error()`. No threshold: it only # has to keep running, and its numbers track the creation stack's cost. - name: JS signals effect benchmark diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index 28570dd1671..acddb3802f0 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -140,7 +140,11 @@ const player = new Watch.Player({ Pass a signal from [`@moq/signals`](/lib/js/signals) for any control you want to change later, such as `muted` or `delay`. `Player` is the pipeline inside -``; its parts are exported for custom composition. +``; its parts are exported for custom composition. The audio and video +decoders expose a recent `arrivals` window for timeline overlays. It keeps transport +ingress timestamps through buffering and distinguishes late frames from latency +skips. Read the window as a snapshot, since reactive updates can coalesce; it is +bounded history rather than a cumulative statistics source. ## Buffered playback diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 0443a095d38..ef19b1a130b 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -1529,7 +1529,10 @@ test("Consumer reports a marker group as a playhead event", async () => { expect(reset?.frame).toBeUndefined(); expect(reset?.discontinuity).toBe(1); expect(reset?.continuous).toBe(false); - expect((await nextFrame(consumer))?.discontinuity).toBe(1); + expect(reset?.skipped).toBe(false); + const afterMarker = await nextFrame(consumer); + expect(afterMarker?.discontinuity).toBe(1); + expect(afterMarker?.skipped).toBe(false); consumer.close(); }); @@ -1567,6 +1570,7 @@ test("Consumer latency skip bumps playhead generation once", async () => { const resumed = await nextFrame(consumer); expect(resumed?.frame?.timestamp).toBe(100_000 as Time.Micro); expect(resumed?.discontinuity).toBe(1); + expect(resumed?.skipped).toBe(true); consumer.close(); }); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index a7e563d6fd2..f4f754810dd 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -4,7 +4,7 @@ import { Effect, type Getter, type GetterInit, getter, Once, Signal } from "@moq import type { Format } from "./format"; import { Jitter } from "./jitter"; -import type { BufferedRanges, Frame } from "./types"; +import type { BufferedRanges, ReceivedFrame } from "./types"; /** Options for constructing a {@link Consumer}; the obsolete `maxAge` prop is refused. */ export interface ConsumerProps { @@ -23,7 +23,7 @@ export interface ConsumerProps { interface Group { consumer: Moq.Group.Consumer; - frames: Frame[]; // decode order + frames: ReceivedFrame[]; // decode order empty: boolean; // no wire frame was published; empty groups mean nothing media: boolean; // a decodable (non-marker) frame was buffered start?: Time.Micro; // First decoded timestamp @@ -87,6 +87,7 @@ export class Consumer { // sits. Only the consumer can know this, which is why next() reports it instead of leaving // callers to guess from group numbers. #gap = false; + #skipped = false; // The first media timestamp of the latest delivered group, and its predecessor's start. // Endpoints and later frames can overlap the next group without moving this floor. #start?: { group: number; timestamp: Time.Micro }; @@ -199,7 +200,8 @@ export class Consumer { this.#jitter.observe(arrival, Moq.Time.Milli.fromMicro(sample.timestamp)); this.#spread.set(this.#jitter.measured); } - const frame: Frame = { + const frame: ReceivedFrame = { + arrival, payload: sample.payload, timestamp: sample.timestamp, // Protocol invariant: groups always start at a keyframe. @@ -362,6 +364,7 @@ export class Consumer { if (marker || !ptsContiguous(first.end ?? this.#presentedEnd, nextStart)) { hole = true; } + if (!marker && (!first.empty || !first.done)) this.#skipped = true; first.consumer.close(); first.frames.length = 0; skipped++; @@ -462,10 +465,12 @@ export class Consumer { */ async next(): Promise< | { - frame: Frame | undefined; + frame: ReceivedFrame | undefined; group: number; discontinuity: number; continuous: boolean; + /** Whether latency enforcement skipped content before this media frame. */ + skipped: boolean; end?: Time.Micro; } | undefined @@ -514,6 +519,7 @@ export class Consumer { if (!startup && (skipHole || ended !== undefined) && !contiguous && !empty) this.#markPlayhead(); if (!contiguous) this.#gap = true; + if (!startup && skipHole && !contiguous && !empty) this.#skipped = true; this.#active = head.consumer.sequence; } } @@ -536,6 +542,7 @@ export class Consumer { group: seq, discontinuity: this.#discontinuity, continuous: this.#continuesDelivery(seq), + skipped: false, end, }; } @@ -553,7 +560,9 @@ export class Consumer { throw this.#error; } this.#updateBuffered(); - return { frame, group: seq, discontinuity: this.#discontinuity, continuous }; + const skipped = this.#skipped; + this.#skipped = false; + return { frame, group: seq, discontinuity: this.#discontinuity, continuous, skipped }; } // Check if the group is done and then remove it. A group is removable only @@ -585,6 +594,7 @@ export class Consumer { // A marker carries no content of its own, so this just reports whether // the group it closes was itself reached without a gap. continuous: this.#continuesDelivery(seq), + skipped: false, }; } } diff --git a/js/hang/src/container/types.ts b/js/hang/src/container/types.ts index 1bab11a26ae..02e6579cb6f 100644 --- a/js/hang/src/container/types.ts +++ b/js/hang/src/container/types.ts @@ -17,6 +17,12 @@ export interface Frame { duration?: Time.Micro; } +/** A media frame carrying its local transport ingress time. */ +export interface ReceivedFrame extends Frame { + /** Monotonic ingress time in milliseconds, before reordering or decoder backpressure. */ + arrival: Time.Milli; +} + /** A contiguous span of buffered media, in milliseconds. */ export interface BufferedRange { /** Start of the range in milliseconds. */ diff --git a/js/justfile b/js/justfile index 6baf0524995..6541d29687c 100644 --- a/js/justfile +++ b/js/justfile @@ -81,3 +81,8 @@ build: wasm: cargo build --locked -p moq-wasm --target wasm32-unknown-unknown --profile wasm-release wasm-bindgen --target web --out-name moq --out-dir wasm/dist "${CARGO_TARGET_DIR:-../target}/wasm32-unknown-unknown/wasm-release/moq_wasm.wasm" + +# Sweep decoder count and retained arrival frames independently. +arrivals-bench: + bun install --frozen-lockfile + bun watch/bench/arrivals.ts diff --git a/js/watch/bench/arrivals.ts b/js/watch/bench/arrivals.ts new file mode 100644 index 00000000000..482a40f52a9 --- /dev/null +++ b/js/watch/bench/arrivals.ts @@ -0,0 +1,27 @@ +import { Time } from "@moq/net"; +import { Signal } from "@moq/signals"; +import { type Arrival, recordArrival } from "../src/arrivals"; + +console.log("decoders,retained_frames,ns_per_arrival"); +for (const count of [1, 4, 16]) { + for (const retained of [1, 256, 4096]) { + const initial: Arrival[] = Array.from({ length: retained }, () => ({ + timestamp: Time.Milli(0), + arrival: Time.Milli(0), + late: false, + skipped: false, + })); + const windows = Array.from({ length: count }, () => new Signal(initial)); + const frame = initial[0]; + const start = performance.now(); + for (let n = 0; n < 1000; n++) + for (const window of windows) { + window.set(initial); + recordArrival(window, frame); + } + const ns = ((performance.now() - start) * 1e6) / (1000 * count); + for (const window of windows) + if (window.peek().length !== Math.min(retained + 1, 4096)) throw new Error("lost arrival"); + console.log(`${count},${retained},${ns.toFixed(0)}`); + } +} diff --git a/js/watch/src/arrivals.test.ts b/js/watch/src/arrivals.test.ts new file mode 100644 index 00000000000..cd5d3133c6a --- /dev/null +++ b/js/watch/src/arrivals.test.ts @@ -0,0 +1,30 @@ +import { expect, test } from "bun:test"; +import { Time } from "@moq/net"; +import { Signal } from "@moq/signals"; +import { type Arrival, recordArrival } from "./arrivals"; + +const frame = (arrival: number): Arrival => ({ + timestamp: Time.Milli(arrival), + arrival: Time.Milli(arrival), + late: false, + skipped: false, +}); + +test("coalesced arrivals remain in immutable bounded snapshots", () => { + const window = new Signal([]); + recordArrival(window, frame(0)); + const before = window.peek(); + for (let i = 1; i < 4100; i++) recordArrival(window, frame(i)); + expect(before).toEqual([frame(0)]); + expect(window.peek()).toHaveLength(4096); + expect(window.peek()[0].arrival).toBe(Time.Milli(4)); +}); + +test("time eviction uses newest ingress even when delivery is reordered", () => { + const window = new Signal([]); + recordArrival(window, frame(1000)); + recordArrival(window, frame(32000)); + recordArrival(window, frame(3000)); + recordArrival(window, frame(0)); + expect(window.peek()).toEqual([frame(32000), frame(3000)]); +}); diff --git a/js/watch/src/arrivals.ts b/js/watch/src/arrivals.ts new file mode 100644 index 00000000000..8927ccb99d3 --- /dev/null +++ b/js/watch/src/arrivals.ts @@ -0,0 +1,25 @@ +import { Time } from "@moq/net"; +import type { Signal } from "@moq/signals"; + +/** One received frame, including a latency skip immediately before it. */ +export interface Arrival { + /** Media presentation time in milliseconds. */ + timestamp: Time.Milli; + /** Monotonic transport ingress time in milliseconds. */ + arrival: Time.Milli; + /** Whether ingress missed the playback deadline. */ + late: boolean; + /** Whether latency enforcement skipped content before this frame. */ + skipped: boolean; +} + +/** Retain up to 30 seconds and 4096 frames without losing coalesced signal updates. @internal */ +export function recordArrival(window: Signal, frame: Arrival): void { + window.update((previous) => { + let newest = frame.arrival; + for (const entry of previous) newest = Time.Milli.max(newest, entry.arrival); + const recent = previous.filter((entry) => entry.arrival >= newest - 30_000); + if (frame.arrival >= newest - 30_000) recent.push(frame); + return recent.slice(-4096); + }); +} diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 66d8ed595ec..dac3c338984 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -76,7 +76,13 @@ function frame(): Read { group: 0, discontinuity: 0, continuous: true, - frame: { timestamp: Time.Micro(frameTimestamp), payload: new Uint8Array([1]), keyframe: true }, + skipped: false, + frame: { + arrival: Time.Milli.now(), + timestamp: Time.Micro(frameTimestamp), + payload: new Uint8Array([1]), + keyframe: true, + }, }; } @@ -230,6 +236,7 @@ async function play(initial: Delay) { push, enabled, context: decoder.out.context, + arrivals: decoder.out.arrivals, play, // Remove the rendition from the catalog, then restore it. remove: () => catalog.set({}), @@ -279,6 +286,11 @@ it("decodes an audio packet inside a group after a playhead discontinuity", asyn playback.push(next); await microtasks(); expect(playback.inserted()).toContain(next.frame.timestamp); + expect(playback.arrivals.peek().at(-1)).toMatchObject({ + timestamp: Time.Milli.fromMicro(next.frame.timestamp), + arrival: next.frame.arrival, + skipped: false, + }); } finally { playback.close(); await microtasks(); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 863e48af3d9..ac3cbaca386 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -13,6 +13,7 @@ import { readonlys, Signal, } from "@moq/signals"; +import { type Arrival, recordArrival } from "../arrivals"; import { hostedAssets } from "../assets"; import { base64ToBytes } from "../base64"; import { nextMedia, subscribeMedia } from "../media"; @@ -44,6 +45,8 @@ export type DecoderProps = Inputs & { }; type DecoderOutput = { + /** Recent ingress diagnostics, bounded to 30 seconds and 4096 frames. */ + arrivals: Signal; context: Signal; // The root of the audio graph, which can be used for custom visualizations. @@ -92,6 +95,7 @@ export class Decoder { readonly sync: Sync; readonly #out: DecoderOutput = { + arrivals: new Signal([]), context: new Signal(undefined), root: new Signal(undefined), sampleRate: new Signal(undefined), @@ -340,6 +344,7 @@ export class Decoder { // Every handle to one instance shares `closed`, so it identifies the instance where the // handle itself does not. if (this.#instance !== undefined && this.#instance !== active.closed) { + this.#out.arrivals.set([]); this.#ring.peek()?.reset(); this.#decodeBuffered.set([]); this.sync.reset(); @@ -445,9 +450,10 @@ export class Decoder { const { frame } = next; if (!frame) continue; - // Mark that we received this frame right now. + // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro); - this.sync.received(timestamp, "audio"); + const late = this.sync.received(timestamp, "audio", frame.arrival); + recordArrival(this.#out.arrivals, { timestamp, arrival: frame.arrival, late, skipped: next.skipped }); const duration = packetDuration(config.codec, frame); if (duration !== undefined) this.#frame.set(duration); @@ -548,7 +554,8 @@ export class Decoder { if (!frame) continue; const timestamp = Time.Milli.fromMicro(frame.timestamp); - this.sync.received(timestamp, "audio"); + const late = this.sync.received(timestamp, "audio", frame.arrival); + recordArrival(this.#out.arrivals, { timestamp, arrival: frame.arrival, late, skipped: next.skipped }); const duration = packetDuration(config.codec, frame); if (duration !== undefined) this.#frame.set(duration); diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts index e40ac71be16..08626238bf5 100644 --- a/js/watch/src/audio/replay.test.ts +++ b/js/watch/src/audio/replay.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, type Mock, spyOn } from "bun:test"; import type * as Catalog from "@moq/hang/catalog"; import { Time } from "@moq/net"; -import { type Arrival, type Options, QUANTUM, replay } from "./replay"; +import { type Arrival, type Options, QUANTUM, type Quantum, replay } from "./replay"; const RATE = 48000; const CONFIG = { codec: "opus", sampleRate: RATE, numberOfChannels: 1 } as Catalog.AudioConfig; @@ -134,3 +134,33 @@ describe.each(["shared", "post"] as const)("%s ring at auto", (ring) => { expect(await settled(60, (i) => 30 + (4 - (i % 5)) * 20)).toBeGreaterThanOrEqual(100); }); }); + +describe.each(["shared", "post"] as const)("%s ingress diagnostics", (ring) => { + it("keeps recorded arrival and lateness when delivery is held", async () => { + const trace = [0, 20, 40].map((timestamp) => ({ timestamp, at: timestamp + 1000, group: 0 })); + const run = async (decodeAfter?: number) => { + let window: Quantum["arrivals"] = []; + for await (const quantum of replay(trace, { + ring, + rate: RATE, + config: CONFIG, + delay: Time.Milli(100), + duration: 300, + decodeAfter, + })) + window = quantum.arrivals; + return window; + }; + const direct = await run(); + const held = await run(1200); + expect(held).toEqual(direct); + expect(held).toEqual( + trace.map((entry) => ({ + timestamp: Time.Milli(entry.timestamp), + arrival: Time.Milli(entry.at), + late: false, + skipped: false, + })), + ); + }); +}); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index 3c18faf97ba..c55d6562580 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -18,7 +18,8 @@ import type * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; import { Group, Time, Track } from "@moq/net"; -import { Effect } from "@moq/signals"; +import { Effect, Signal } from "@moq/signals"; +import { type Arrival as Diagnostic, recordArrival } from "../arrivals"; import { type Delay, Sync } from "../sync"; import { frameDuration } from "./config"; import { ringSamples, target } from "./latency"; @@ -50,6 +51,8 @@ export interface Arrival { /** What a replay plays through. */ export interface Options { + /** Hold decoder delivery until this simulated monotonic time. */ + decodeAfter?: number; /** `shared` is the SharedArrayBuffer ring a cross-origin isolated page runs; `post` is the rest. */ ring: "shared" | "post"; /** The sample rate, in Hz. */ @@ -67,6 +70,8 @@ export interface Options { /** One render quantum, as the worklet produced it. */ export interface Quantum { + /** The diagnostic window as of this render quantum. */ + arrivals: readonly Diagnostic[]; /** When it finished rendering, on the trace's clock, in ms. */ at: number; /** The quantum: what the ring supplied at the front, zeros after it. Reused between yields. */ @@ -215,15 +220,22 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato // The decoder's read loop, with the codec taken out. A failure is rethrown at the next quantum. const terminal = new Terminal(); + const arrivals = new Signal([]); + const delivery = Promise.withResolvers(); + if (options.decodeAfter === undefined) delivery.resolve(); let failure: { error: unknown } | undefined; const decode = (async () => { + await delivery.promise; for (;;) { const next = await consumer.next(); if (!next) return; if (terminal.update(next)) ring.reset(); if (next.end !== undefined || !next.frame) continue; - const { timestamp } = next.frame; + const { timestamp, arrival } = next.frame; + const media = Time.Milli.fromMicro(timestamp); + const late = sync.received(media, "audio", arrival); + recordArrival(arrivals, { timestamp: media, arrival, late, skipped: next.skipped }); const length = samples.get(timestamp); if (length === undefined) throw new Error(`the consumer delivered an unrecorded frame at ${timestamp} us`); ring.insert(timestamp, [new Float32Array(length).fill(0.5)]); @@ -275,6 +287,11 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato next++; } if (next > stamped) await settle(); + clock = now; + if (options.decodeAfter !== undefined && now >= options.decodeAfter) { + delivery.resolve(); + await settle(); + } if (failure) throw failure.error; // The decoder resizes the ring as the resolved delay moves, which only an arrival does. @@ -286,9 +303,17 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato output.fill(0); ring.read([output]); - yield { at: now, output, stalled: ring.stalled, timestamp: Time.Milli.fromMicro(ring.timestamp), delay }; + yield { + arrivals: arrivals.peek(), + at: now, + output, + stalled: ring.stalled, + timestamp: Time.Milli.fromMicro(ring.timestamp), + delay, + }; } } finally { + delivery.resolve(); consumer.close(); track.close(); await decode; diff --git a/js/watch/src/index.ts b/js/watch/src/index.ts index 3b2eb13a433..1bbbdc9fbc3 100644 --- a/js/watch/src/index.ts +++ b/js/watch/src/index.ts @@ -11,6 +11,7 @@ export * as Hang from "@moq/hang"; export type { BufferedRange, BufferedRanges } from "@moq/hang/container"; export * as Net from "@moq/net"; export * as Signals from "@moq/signals"; +export type { Arrival } from "./arrivals"; export { assets } from "./assets"; export * as Audio from "./audio"; export * from "./broadcast"; diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index d0bef3018d8..d4cd9ad00f6 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -159,3 +159,21 @@ describe("wait", () => { sync.close(); }); }); + +describe("recorded ingress", () => { + it("anchors and classifies using arrival rather than delayed delivery", async () => { + const sync = new Sync({ delay: Time.Milli(100) }); + const original = performance.now; + performance.now = () => 10000; + try { + for (let i = 0; i < 20; i++) await Promise.resolve(); + expect(sync.received(Time.Milli(0), "audio", Time.Milli(1000))).toBe(false); + expect(sync.out.reference.peek()).toBe(Time.Milli(1000)); + expect(sync.received(Time.Milli(20), "audio", Time.Milli(1020))).toBe(false); + expect(sync.received(Time.Milli(40), "audio", Time.Milli(1200))).toBe(true); + } finally { + performance.now = original; + sync.close(); + } + }); +}); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index 048572cf574..4247fdc13d3 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -144,18 +144,18 @@ export class Sync { this.#out.delay.set(delay); } + /** Anchor playback from ingress time and report whether the frame arrived late. */ // Fold a newly received frame into the reference. The reference anchors playback to the // wall clock; we lower it (skip ahead) only when keeping it would push the lookahead past `maxDelay`. - received(timestamp: Time.Milli, label = ""): void { + received(timestamp: Time.Milli, label = "", arrival = Time.Milli.now()): boolean { this.#out.timestamp.update((current) => (current === undefined || timestamp > current ? timestamp : current)); - const now = Time.Milli.now(); - const ref = Time.Milli.sub(now, timestamp); + const ref = Time.Milli.sub(arrival, timestamp); const currentRef = this.#out.reference.peek(); // First frame anchors the reference. if (currentRef === undefined) { this.#out.reference.set(ref); - return; + return false; } // Check if `wait()` would not sleep at all. @@ -182,15 +182,16 @@ export class Sync { } // Frame isn't earlier than the anchor: it can't add lookahead, so keep the reference. - if (ref >= currentRef) return; + if (ref >= currentRef) return sleep < 0; // Frame is earlier (more lookahead). `sleep` is how far ahead of the playhead keeping the // anchor would put it. const cap = this.#out.maxDelay.peek(); - if (sleep <= cap) return; // within budget: let the buffer grow instead of skipping ahead + if (sleep <= cap) return sleep < 0; // within budget: let the buffer grow instead of skipping ahead // Over the cap: re-anchor down so the resulting lookahead is exactly the cap. this.#out.reference.set(Time.Milli.add(ref, Time.Milli.sub(cap, delay))); + return sleep < 0; } // Re-anchor playback to the next frame received. Call this at an utterance boundary diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 71ba45a02c2..e041b817e34 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -179,7 +179,8 @@ function sample(group: number, timestamp: number, discontinuity = 0, keyframe = group, discontinuity, continuous: true, - frame: { timestamp: Time.Micro(timestamp), payload: new Uint8Array([1]), keyframe }, + skipped: false, + frame: { arrival: Time.Milli.now(), timestamp: Time.Micro(timestamp), payload: new Uint8Array([1]), keyframe }, }; } @@ -408,7 +409,7 @@ it("clears the picture once no rendition is selectable", async () => { // A late marker from older history raises a discontinuity without making older groups newer. function marker(group: number, end: number, discontinuity: number): Read { - return { group, discontinuity, continuous: false, frame: undefined, end: Time.Micro(end) }; + return { group, discontinuity, continuous: false, skipped: false, frame: undefined, end: Time.Micro(end) }; } for (const kind of ["legacy", "cmaf"] as const) { @@ -423,6 +424,11 @@ for (const kind of ["legacy", "cmaf"] as const) { ]); try { expect(playback.submitted).toEqual([1000, 1100, 1200]); + expect(playback.decoder.out.arrivals.peek().map((entry) => entry.timestamp)).toEqual([ + Time.Milli(1), + Time.Milli(1.1), + Time.Milli(1.2), + ]); // Rejected payloads were still received, but never counted as frames. expect(playback.decoder.out.stats.peek()).toEqual({ frameCount: 3, bytesReceived: 5 }); } finally { diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 9a4986e54d1..b0a3b34aade 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -14,6 +14,7 @@ import { Signal, } from "@moq/signals"; import { rotateVideoDimensions } from "@moq/video"; +import { type Arrival, recordArrival } from "../arrivals"; import { base64ToBytes } from "../base64"; import { nextMedia, subscribeMedia } from "../media"; @@ -54,6 +55,8 @@ export interface Stats { } type DecoderOutput = { + /** Recent ingress diagnostics, bounded to 30 seconds and 4096 frames. */ + arrivals: Signal; // The current frame to render. frame: Signal; @@ -80,6 +83,7 @@ export class Decoder { readonly sync: Sync; readonly #out: DecoderOutput = { + arrivals: new Signal([]), frame: new Signal(undefined), timestamp: new Signal(undefined), display: new Signal<{ width: number; height: number } | undefined>(undefined), @@ -196,6 +200,7 @@ export class Decoder { // it to catch up to the old picture would freeze video until it passed the old timestamps, // so switch at once and re-anchor the clock. A rendition switch within one instance, or a // switch to another path, still waits. + if (this.#instance && this.#instance.closed !== active.closed) this.#out.arrivals.set([]); const republished = this.#instance?.path === active.path && this.#instance.closed !== active.closed; this.#instance = { path: active.path, closed: active.closed }; if (republished) this.sync.reset(); @@ -260,6 +265,7 @@ export class Decoder { }); }); effect.proxy(this.#out.buffered, active.buffered); + effect.proxy(this.#out.arrivals, active.arrivals); effect.proxy(this.#spread, active.spread); } @@ -328,6 +334,7 @@ class DecoderTrack { config: DecoderConfig; catalog: Getter; stats: Signal; + arrivals = new Signal([]); timestamp = new Signal(undefined); frame = new Signal(undefined); @@ -489,9 +496,10 @@ class DecoderTrack { })); if (stale) continue; - // Mark that we received this frame right now. + // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro); - this.sync.received(timestamp, "video"); + const late = this.sync.received(timestamp, "video", frame.arrival); + recordArrival(this.arrivals, { timestamp, arrival: frame.arrival, late, skipped: next.skipped }); const chunk = new EncodedVideoChunk({ type: frame.keyframe ? "key" : "delta", @@ -569,9 +577,10 @@ class DecoderTrack { })); if (stale) continue; - // Mark that we received this frame right now. + // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Time.Milli.fromMicro(frame.timestamp); - this.sync.received(timestamp, "video"); + const late = this.sync.received(timestamp, "video", frame.arrival); + recordArrival(this.arrivals, { timestamp, arrival: frame.arrival, late, skipped: next.skipped }); // Track decode buffer (see #runLegacy: bridge on the consumer's continuity signal, // never on group adjacency, which proves nothing about the timeline). From 3c4ca61a515336d860ef45f5d71979d634d6ead5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 11 Oct 2026 18:10:57 -0700 Subject: [PATCH 06/19] refactor(watch): replace retained arrival windows with caller-owned handles Decoders no longer keep an `out.arrivals` snapshot. `arrivals()` opens a handle that queues every frame received while it is open; closing it releases the queue, so players that don't chart retain nothing. Video reports only the rendition on screen. The quest is complete and removed. Co-Authored-By: Claude Opus 5.5 --- doc/lib/js/watch.md | 22 +++++-- js/justfile | 2 +- js/watch/bench/arrivals.ts | 44 +++++++------- js/watch/src/arrivals.test.ts | 61 +++++++++++++------ js/watch/src/arrivals.ts | 95 ++++++++++++++++++++++++++---- js/watch/src/audio/decoder.test.ts | 8 ++- js/watch/src/audio/decoder.ts | 19 +++--- js/watch/src/audio/replay.ts | 12 ++-- js/watch/src/index.ts | 2 +- js/watch/src/video/decoder.test.ts | 10 ++-- js/watch/src/video/decoder.ts | 25 +++++--- quest/m1/README.md | 1 - quest/m1/watch-arrivals.md | 44 -------------- 13 files changed, 214 insertions(+), 131 deletions(-) delete mode 100644 quest/m1/watch-arrivals.md diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index 3357760576d..f92c59dee08 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -146,11 +146,23 @@ const player = new Watch.Player({ Pass a signal from [`@moq/signals`](/lib/js/signals) for any control you want to change later, such as `muted` or `delay`. `Player` is the pipeline inside -``; its parts are exported for custom composition. The audio and video -decoders expose a recent `arrivals` window for timeline overlays. It keeps transport -ingress timestamps through buffering and distinguishes late frames from latency -skips. Read the window as a snapshot, since reactive updates can coalesce; it is -bounded history rather than a cumulative statistics source. +``; its parts are exported for custom composition. + +To chart when each frame arrived against when it plays, open an `arrivals()` +handle on a decoder. It yields every frame received while it is open, stamped +when the frame came off the transport: + +```ts +const arrivals = player.audio.arrivals(); +for await (const { timestamp, arrival, late, skipped } of arrivals) { + // `late`: arrived after its playback deadline. + // `skipped`: content right before this frame was dropped to hold the latency budget. +} +``` + +Unread frames queue on the handle, so keep reading and keep your own window. +`arrivals.close()` stops observing and releases them; nothing is retained +without an open handle. Video reports the rendition on screen. ## Buffered playback diff --git a/js/justfile b/js/justfile index a410e59c6b9..84cbbee06b8 100644 --- a/js/justfile +++ b/js/justfile @@ -96,7 +96,7 @@ qmux-bench: bunx tsc --noEmit --project net/bench/tsconfig.json bun net/bench/qmux.ts -# Sweep decoder count and retained arrival frames independently. +# Sweep open arrival handles against their unread backlog. arrivals-bench: bun install --frozen-lockfile bun watch/bench/arrivals.ts diff --git a/js/watch/bench/arrivals.ts b/js/watch/bench/arrivals.ts index 482a40f52a9..829684981a2 100644 --- a/js/watch/bench/arrivals.ts +++ b/js/watch/bench/arrivals.ts @@ -1,27 +1,29 @@ import { Time } from "@moq/net"; -import { Signal } from "@moq/signals"; -import { type Arrival, recordArrival } from "../src/arrivals"; +import { Producer } from "../src/arrivals"; -console.log("decoders,retained_frames,ns_per_arrival"); +// Sweep open handles against each handle's unread backlog: a write should cost one push per +// handle, never a slope in the backlog. +const frame = { timestamp: Time.Milli(0), arrival: Time.Milli(0), late: false, skipped: false }; +const WRITES = 10_000; + +console.log("handles,backlog,ns_per_handle_write"); for (const count of [1, 4, 16]) { - for (const retained of [1, 256, 4096]) { - const initial: Arrival[] = Array.from({ length: retained }, () => ({ - timestamp: Time.Milli(0), - arrival: Time.Milli(0), - late: false, - skipped: false, - })); - const windows = Array.from({ length: count }, () => new Signal(initial)); - const frame = initial[0]; + for (const backlog of [0, 256, 4096]) { + const producer = new Producer(); + const handles = Array.from({ length: count }, () => producer.consume()); + for (let i = 0; i < backlog; i++) producer.write(frame); + const start = performance.now(); - for (let n = 0; n < 1000; n++) - for (const window of windows) { - window.set(initial); - recordArrival(window, frame); - } - const ns = ((performance.now() - start) * 1e6) / (1000 * count); - for (const window of windows) - if (window.peek().length !== Math.min(retained + 1, 4096)) throw new Error("lost arrival"); - console.log(`${count},${retained},${ns.toFixed(0)}`); + for (let n = 0; n < WRITES; n++) { + producer.write(frame); + for (const handle of handles) await handle.next(); + } + const ns = ((performance.now() - start) * 1e6) / (WRITES * count); + + for (const handle of handles) { + for (let i = 0; i < backlog; i++) if ((await handle.next()) === undefined) throw new Error("lost arrival"); + handle.close(); + } + console.log(`${count},${backlog},${ns.toFixed(0)}`); } } diff --git a/js/watch/src/arrivals.test.ts b/js/watch/src/arrivals.test.ts index cd5d3133c6a..a64587c0d57 100644 --- a/js/watch/src/arrivals.test.ts +++ b/js/watch/src/arrivals.test.ts @@ -1,30 +1,55 @@ import { expect, test } from "bun:test"; import { Time } from "@moq/net"; -import { Signal } from "@moq/signals"; -import { type Arrival, recordArrival } from "./arrivals"; +import { Producer } from "./arrivals"; -const frame = (arrival: number): Arrival => ({ +const frame = (arrival: number) => ({ timestamp: Time.Milli(arrival), arrival: Time.Milli(arrival), late: false, skipped: false, }); -test("coalesced arrivals remain in immutable bounded snapshots", () => { - const window = new Signal([]); - recordArrival(window, frame(0)); - const before = window.peek(); - for (let i = 1; i < 4100; i++) recordArrival(window, frame(i)); - expect(before).toEqual([frame(0)]); - expect(window.peek()).toHaveLength(4096); - expect(window.peek()[0].arrival).toBe(Time.Milli(4)); +test("a handle receives every frame written while it is open", async () => { + const producer = new Producer(); + producer.write(frame(0)); + const handle = producer.consume(); + for (let i = 1; i <= 3; i++) producer.write(frame(i)); + for (let i = 1; i <= 3; i++) expect(await handle.next()).toEqual(frame(i)); + + const waiting = handle.next(); + producer.write(frame(4)); + expect(await waiting).toEqual(frame(4)); + handle.close(); +}); + +test("closing a handle releases its unread frames and stops observing", async () => { + const producer = new Producer(); + const closed = producer.consume(); + const open = producer.consume(); + producer.write(frame(0)); + closed.close(); + producer.write(frame(1)); + expect(await closed.next()).toBeUndefined(); + expect(await open.next()).toEqual(frame(0)); + expect(await open.next()).toEqual(frame(1)); + open.close(); +}); + +test("a closed decoder ends its handles once they drain", async () => { + const producer = new Producer(); + const handle = producer.consume(); + producer.write(frame(0)); + producer.close(); + expect(await handle.next()).toEqual(frame(0)); + expect(await handle.next()).toBeUndefined(); + expect(await producer.consume().next()).toBeUndefined(); }); -test("time eviction uses newest ingress even when delivery is reordered", () => { - const window = new Signal([]); - recordArrival(window, frame(1000)); - recordArrival(window, frame(32000)); - recordArrival(window, frame(3000)); - recordArrival(window, frame(0)); - expect(window.peek()).toEqual([frame(32000), frame(3000)]); +test("a pending read refuses a concurrent one", async () => { + const producer = new Producer(); + const handle = producer.consume(); + const waiting = handle.next(); + await expect(handle.next()).rejects.toThrow("multiple calls to next not supported"); + handle.close(); + expect(await waiting).toBeUndefined(); }); diff --git a/js/watch/src/arrivals.ts b/js/watch/src/arrivals.ts index 8927ccb99d3..9684b46788c 100644 --- a/js/watch/src/arrivals.ts +++ b/js/watch/src/arrivals.ts @@ -1,5 +1,4 @@ -import { Time } from "@moq/net"; -import type { Signal } from "@moq/signals"; +import type { Time } from "@moq/net"; /** One received frame, including a latency skip immediately before it. */ export interface Arrival { @@ -13,13 +12,87 @@ export interface Arrival { skipped: boolean; } -/** Retain up to 30 seconds and 4096 frames without losing coalesced signal updates. @internal */ -export function recordArrival(window: Signal, frame: Arrival): void { - window.update((previous) => { - let newest = frame.arrival; - for (const entry of previous) newest = Time.Milli.max(newest, entry.arrival); - const recent = previous.filter((entry) => entry.arrival >= newest - 30_000); - if (frame.arrival >= newest - 30_000) recent.push(frame); - return recent.slice(-4096); - }); +interface State { + queue: Arrival[]; + closed: boolean; + wake?: () => void; +} + +/** + * Every frame a decoder receives while this handle is open, in delivery order. + * + * Unread entries queue until {@link next} takes them, so read continuously or close the handle; + * closing releases the queue. Obtain one from a decoder's `arrivals()`. + */ +export class Arrivals implements AsyncIterable { + readonly #state: State; + readonly #open: Set; + + /** @internal */ + constructor(state: State, open: Set) { + this.#state = state; + this.#open = open; + } + + /** Get the next received frame, or `undefined` once the handle or its decoder closes. */ + async next(): Promise { + const state = this.#state; + for (;;) { + const entry = state.queue.shift(); + if (entry) return entry; + if (state.closed) return undefined; + if (state.wake) throw new Error("multiple calls to next not supported"); + await new Promise((resolve) => { + state.wake = resolve; + }); + state.wake = undefined; + } + } + + /** Iterate received frames until the handle or its decoder closes. */ + async *[Symbol.asyncIterator](): AsyncIterator { + for (;;) { + const entry = await this.next(); + if (entry === undefined) return; + yield entry; + } + } + + /** Stop observing and release unread entries. */ + close(): void { + this.#open.delete(this.#state); + this.#state.queue.length = 0; + this.#state.closed = true; + this.#state.wake?.(); + } +} + +/** Fans each received frame out to the open {@link Arrivals} handles. @internal */ +export class Producer { + readonly #open = new Set(); + #closed = false; + + /** Open a handle that receives every later frame. */ + consume(): Arrivals { + const state: State = { queue: [], closed: this.#closed }; + if (!this.#closed) this.#open.add(state); + return new Arrivals(state, this.#open); + } + + write(arrival: Arrival): void { + for (const state of this.#open) { + state.queue.push(arrival); + state.wake?.(); + } + } + + /** End every handle once it drains what it already received. */ + close(): void { + this.#closed = true; + for (const state of this.#open) { + state.closed = true; + state.wake?.(); + } + this.#open.clear(); + } } diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 5997fd568ff..3f6606dbdca 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -253,7 +253,7 @@ async function play(initial: Delay, buffered = false, cmaf = false) { enabled, decoding, context: decoder.out.context, - arrivals: decoder.out.arrivals, + arrivals: () => decoder.arrivals(), error: decoder.out.error, play, // Remove the rendition from the catalog, then restore it. @@ -302,6 +302,8 @@ it("decodes an audio packet inside a group after a playhead discontinuity", asyn const playback = await play(Time.Milli(100)); try { await playback.play(); + // Opened late, so it observes only the frame below. + const arrivals = playback.arrivals(); const next = frame(); next.discontinuity = 1; if (!next.frame) throw new Error("missing test frame"); @@ -309,11 +311,13 @@ it("decodes an audio packet inside a group after a playhead discontinuity", asyn playback.push(next); await microtasks(); expect(playback.inserted()).toContain(next.frame.timestamp); - expect(playback.arrivals.peek().at(-1)).toMatchObject({ + expect(await arrivals.next()).toMatchObject({ timestamp: Time.Milli.fromMicro(next.frame.timestamp), arrival: next.frame.arrival, skipped: false, }); + arrivals.close(); + expect(await arrivals.next()).toBeUndefined(); } finally { playback.close(); await microtasks(); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 5fa18297b4e..01fc7cbb53e 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -13,7 +13,7 @@ import { readonlys, Signal, } from "@moq/signals"; -import { type Arrival, recordArrival } from "../arrivals"; +import { Producer as ArrivalProducer, type Arrivals } from "../arrivals"; import { hostedAssets } from "../assets"; import { base64ToBytes } from "../base64"; import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media"; @@ -45,8 +45,6 @@ export type DecoderProps = Inputs & { }; type DecoderOutput = { - /** Recent ingress diagnostics, bounded to 30 seconds and 4096 frames. */ - arrivals: Signal; /** Fatal decode failure, cleared when another source is selected. */ error: Signal; context: Signal; @@ -97,7 +95,6 @@ export class Decoder { readonly sync: Sync; readonly #out: DecoderOutput = { - arrivals: new Signal([]), error: new Signal(undefined), context: new Signal(undefined), root: new Signal(undefined), @@ -141,6 +138,9 @@ export class Decoder { // The broadcast instance the ring's timeline belongs to. See #runDecoder. #instance?: Moq.Broadcast.Consumer["closed"]; + // Fans received frames out to open `arrivals()` handles; retains nothing without one. + #arrivals = new ArrivalProducer(); + #signals = new Effect(); #failed?: Failure; @@ -374,7 +374,6 @@ export class Decoder { // Every handle to one instance shares `closed`, so it identifies the instance where the // handle itself does not. if (this.#instance !== undefined && this.#instance !== active.closed) { - this.#out.arrivals.set([]); this.#ring.peek()?.reset(); this.#decodeBuffered.set([]); this.sync.reset(); @@ -490,7 +489,7 @@ export class Decoder { // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro); const late = this.sync.received(timestamp, "audio", frame.arrival); - recordArrival(this.#out.arrivals, { timestamp, arrival: frame.arrival, late, skipped: next.skipped }); + this.#arrivals.write({ timestamp, arrival: frame.arrival, late, skipped: next.skipped }); const duration = packetDuration(config.codec, frame); if (duration !== undefined) this.#frame.set(duration); @@ -597,7 +596,7 @@ export class Decoder { const timestamp = Time.Milli.fromMicro(frame.timestamp); const late = this.sync.received(timestamp, "audio", frame.arrival); - recordArrival(this.#out.arrivals, { timestamp, arrival: frame.arrival, late, skipped: next.skipped }); + this.#arrivals.write({ timestamp, arrival: frame.arrival, late, skipped: next.skipped }); const duration = packetDuration(config.codec, frame); if (duration !== undefined) this.#frame.set(duration); @@ -753,7 +752,13 @@ export class Decoder { return true; } + /** Observe every frame received from now until the handle or this decoder closes. */ + arrivals(): Arrivals { + return this.#arrivals.consume(); + } + close() { + this.#arrivals.close(); this.#signals.close(); } diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index c55d6562580..dcdec0e6058 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -18,8 +18,8 @@ import type * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; import { Group, Time, Track } from "@moq/net"; -import { Effect, Signal } from "@moq/signals"; -import { type Arrival as Diagnostic, recordArrival } from "../arrivals"; +import { Effect } from "@moq/signals"; +import type { Arrival as Diagnostic } from "../arrivals"; import { type Delay, Sync } from "../sync"; import { frameDuration } from "./config"; import { ringSamples, target } from "./latency"; @@ -70,7 +70,7 @@ export interface Options { /** One render quantum, as the worklet produced it. */ export interface Quantum { - /** The diagnostic window as of this render quantum. */ + /** Every frame the decoder received so far, in delivery order. Grows between yields. */ arrivals: readonly Diagnostic[]; /** When it finished rendering, on the trace's clock, in ms. */ at: number; @@ -220,7 +220,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato // The decoder's read loop, with the codec taken out. A failure is rethrown at the next quantum. const terminal = new Terminal(); - const arrivals = new Signal([]); + const arrivals: Diagnostic[] = []; const delivery = Promise.withResolvers(); if (options.decodeAfter === undefined) delivery.resolve(); let failure: { error: unknown } | undefined; @@ -235,7 +235,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const { timestamp, arrival } = next.frame; const media = Time.Milli.fromMicro(timestamp); const late = sync.received(media, "audio", arrival); - recordArrival(arrivals, { timestamp: media, arrival, late, skipped: next.skipped }); + arrivals.push({ timestamp: media, arrival, late, skipped: next.skipped }); const length = samples.get(timestamp); if (length === undefined) throw new Error(`the consumer delivered an unrecorded frame at ${timestamp} us`); ring.insert(timestamp, [new Float32Array(length).fill(0.5)]); @@ -304,7 +304,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato output.fill(0); ring.read([output]); yield { - arrivals: arrivals.peek(), + arrivals, at: now, output, stalled: ring.stalled, diff --git a/js/watch/src/index.ts b/js/watch/src/index.ts index 1bbbdc9fbc3..b8a6a41c6b8 100644 --- a/js/watch/src/index.ts +++ b/js/watch/src/index.ts @@ -11,7 +11,7 @@ export * as Hang from "@moq/hang"; export type { BufferedRange, BufferedRanges } from "@moq/hang/container"; export * as Net from "@moq/net"; export * as Signals from "@moq/signals"; -export type { Arrival } from "./arrivals"; +export type { Arrival, Arrivals } from "./arrivals"; export { assets } from "./assets"; export * as Audio from "./audio"; export * from "./broadcast"; diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 3312ac40ccb..15f60db5182 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -275,9 +275,12 @@ async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: rea const sync = new Sync({ delay }); const enabled = new Signal(true); const decoder = new Decoder({ source, sync, enabled }); + // Opened before the reads below, so it observes every one of them. + const arrivals = decoder.arrivals(); await microtasks(); return { decoder, + arrivals, delay, fail: (error: DOMException, index = -1) => errors.at(index)?.(error), emit: (timestamp: number, index = -1) => frames.at(index)?.(new Picture(timestamp) as unknown as VideoFrame), @@ -438,11 +441,8 @@ for (const kind of ["legacy", "cmaf"] as const) { ]); try { expect(playback.submitted).toEqual([1000, 1100, 1200]); - expect(playback.decoder.out.arrivals.peek().map((entry) => entry.timestamp)).toEqual([ - Time.Milli(1), - Time.Milli(1.1), - Time.Milli(1.2), - ]); + for (const timestamp of [1, 1.1, 1.2]) + expect((await playback.arrivals.next())?.timestamp).toBe(Time.Milli(timestamp)); // Rejected payloads were still received, but never counted as frames. expect(playback.decoder.out.stats.peek()).toEqual({ frameCount: 3, bytesReceived: 5 }); } finally { diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 67ef9285aad..8340b4c1af3 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -14,7 +14,7 @@ import { Signal, } from "@moq/signals"; import { rotateVideoDimensions } from "@moq/video"; -import { type Arrival, recordArrival } from "../arrivals"; +import { Producer as ArrivalProducer, type Arrivals } from "../arrivals"; import { base64ToBytes } from "../base64"; import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media"; @@ -55,8 +55,6 @@ export interface Stats { } type DecoderOutput = { - /** Recent ingress diagnostics, bounded to 30 seconds and 4096 frames. */ - arrivals: Signal; /** Fatal decode failure, cleared when another source is selected. */ error: Signal; // The current frame to render. @@ -85,7 +83,6 @@ export class Decoder { readonly sync: Sync; readonly #out: DecoderOutput = { - arrivals: new Signal([]), error: new Signal(undefined), frame: new Signal(undefined), timestamp: new Signal(undefined), @@ -110,6 +107,9 @@ export class Decoder { #instance?: { path: Moq.Path.Valid; closed: unknown }; readonly #identity: Computed; + // Fans received frames out to open `arrivals()` handles; retains nothing without one. + #arrivals = new ArrivalProducer(); + #signals = new Effect(); #selected?: DecoderTrack; #failed?: Failure; @@ -224,7 +224,6 @@ export class Decoder { // it to catch up to the old picture would freeze video until it passed the old timestamps, // so switch at once and re-anchor the clock. A rendition switch within one instance, or a // switch to another path, still waits. - if (this.#instance && this.#instance.closed !== active.closed) this.#out.arrivals.set([]); const republished = this.#instance?.path === active.path && this.#instance.closed !== active.closed; this.#instance = { path: active.path, closed: active.closed }; if (republished) this.sync.reset(); @@ -274,6 +273,8 @@ export class Decoder { // Upgrade the pending track to active. // #runActive will be in charge of it now. + if (current) current.arrivals = undefined; + pending.arrivals = this.#arrivals; this.#active.set(pending); this.#pending.set(undefined); pending = undefined; @@ -319,7 +320,6 @@ export class Decoder { }); }); effect.proxy(this.#out.buffered, active.buffered); - effect.proxy(this.#out.arrivals, active.arrivals); effect.proxy(this.#spread, active.spread); } @@ -371,8 +371,14 @@ export class Decoder { }, BUFFERING); } + /** Observe every frame the displayed rendition receives until the handle or this decoder closes. */ + arrivals(): Arrivals { + return this.#arrivals.consume(); + } + close() { this.#clearCurrentFrame(); + this.#arrivals.close(); this.#signals.close(); } @@ -403,7 +409,8 @@ class DecoderTrack { identity: string; catalog: Getter; stats: Signal; - arrivals = new Signal([]); + // Set once promoted, so only the rendition feeding the picture reports arrivals. + arrivals?: ArrivalProducer; timestamp = new Signal(undefined); frame = new Signal(undefined); @@ -569,7 +576,7 @@ class DecoderTrack { // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro); const late = this.sync.received(timestamp, "video", frame.arrival); - recordArrival(this.arrivals, { timestamp, arrival: frame.arrival, late, skipped: next.skipped }); + this.arrivals?.write({ timestamp, arrival: frame.arrival, late, skipped: next.skipped }); const chunk = new EncodedVideoChunk({ type: frame.keyframe ? "key" : "delta", @@ -650,7 +657,7 @@ class DecoderTrack { // Classify ingress before decoder backpressure changes the delivery clock. const timestamp = Time.Milli.fromMicro(frame.timestamp); const late = this.sync.received(timestamp, "video", frame.arrival); - recordArrival(this.arrivals, { timestamp, arrival: frame.arrival, late, skipped: next.skipped }); + this.arrivals?.write({ timestamp, arrival: frame.arrival, late, skipped: next.skipped }); // Track decode buffer (see #runLegacy: bridge on the consumer's continuity signal, // never on group adjacency, which proves nothing about the timeline). diff --git a/quest/m1/README.md b/quest/m1/README.md index 43ac7d19607..0f469ed670b 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -59,7 +59,6 @@ session outcomes, and embedder refusals inside session outcomes. - [Cluster routing](/quest/m1/cluster-routing/README.md) - any node routes toward a broadcast's origin over CDN and P2P links alike, with per-origin routes and path-less announces - [Flat questlines](/quest/m1/quest-flat-lines.md) - moq pins the current quest CLI, lands its questline branches on main, and retires them - [Binding audio delay](/quest/m1/binding-surface.md) - moq-ffi and every wrapper configure and observe audio playout delay -- [Per-frame arrivals in @moq/watch](/quest/m1/watch-arrivals.md) - the audio and video decoders expose a window of recent frame arrivals, late and skipped marks included, as a signal - [Bump web-transport-iroh for the capsule close](/quest/m1/iroh-capsule-bump.md) - condition: moq-dev/web-transport#419 ships in a release, then moq's iroh HTTP/3 client reports a peer's close capsule - [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - every stream group in a lite subscription arrives or is dropped by name, and lite-07 drops its stream count for it - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - a lite-07 SUBSCRIBE asks for past and live ranges with absolute or relative ends, in either order, and replaces FETCH; merged ranges never starve a subscriber; relays fill misses by range, including over moq-transport diff --git a/quest/m1/watch-arrivals.md b/quest/m1/watch-arrivals.md deleted file mode 100644 index b0f86f09474..00000000000 --- a/quest/m1/watch-arrivals.md +++ /dev/null @@ -1,44 +0,0 @@ -# [S] Per-frame arrivals in @moq/watch - -## Goal - -The audio and video decoders in `@moq/watch` each expose an `arrivals` -signal: a bounded window of recent frames as `{timestamp, arrival, late, -skipped}`, where `arrival` is when the frame came off the transport, `late` -means it arrived after its sync deadline (not that it rendered late), and -`skipped` marks content the consumer dropped to stay within the latency -budget. A page can draw per-frame arrival against playout without patching -`Sync.received`. - -## Plan - -Decided 2026-10-09: - -- **A signal, not a callback.** A signal holding only the newest arrival - would coalesce and lose frames between reads, so it holds a window that - updates per frame. It must reach back at least as far as the span the - first consumer draws (a transcript word); time or count bound is chosen - when implementing. -- **Feed it from where the data already exists.** `Sync.received()` - computes lateness per frame and only logs it today; the container consumer - knows which groups it skipped and only warns. Neither has a signal. -- **`arrival` is the ingress time.** `Container.Consumer` stamps each frame - as `readFrame()` returns, but the `Frame` it returns drops that stamp, and - `Sync.received()` samples the clock later, after reordering and any - decoder backpressure. Carry the ingress stamp through to the decoder and - compute lateness from it. -- **Skipped content never reaches the decoder.** A latency-skipped group's - frames are not decoded, so they cannot appear as ordinary entries. Decide - how the window represents the gap (a mark on the first frame after it, or - a separate entry), keeping it distinct from other `continuous: false` - causes such as markers. -- The first consumer is moq.dev's Voice AI demo, which draws each word's - frames and marks the ones a WebRTC jitter buffer would have dropped - ([moq.dev's condition quest](https://github.com/moq-dev/moq.dev/blob/main/quest/m1/watch-arrivals-release.md)). -- Test with recorded arrival traces through `js/watch/src/audio/replay.ts`, - including a case where delivery is delayed but the recorded arrival is - unchanged. - -## Related - -- [Browser stats](/quest/m1/stats/js.md) - its late-frame and skip counters share these sources, but count at the producer since a reader of an evicting window can miss entries From 76eb7a66c5bc6781d06556e021151d47f4bfb184 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 11 Oct 2026 18:19:37 -0700 Subject: [PATCH 07/19] fix(watch): close arrival handles when iteration stops Leaving a for-await loop now closes the handle, so it stops queueing frames. A second next() is refused before it can take a frame queued for the pending read. Co-Authored-By: Claude Opus 5.5 --- doc/lib/js/watch.md | 4 ++-- js/watch/src/arrivals.test.ts | 17 +++++++++++++++- js/watch/src/arrivals.ts | 38 ++++++++++++++++++++++------------- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index f92c59dee08..b49d5956376 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -161,8 +161,8 @@ for await (const { timestamp, arrival, late, skipped } of arrivals) { ``` Unread frames queue on the handle, so keep reading and keep your own window. -`arrivals.close()` stops observing and releases them; nothing is retained -without an open handle. Video reports the rendition on screen. +Leaving the loop or calling `arrivals.close()` stops observing and releases +them; nothing is retained without an open handle. Video reports the rendition on screen. ## Buffered playback diff --git a/js/watch/src/arrivals.test.ts b/js/watch/src/arrivals.test.ts index a64587c0d57..2559a26954a 100644 --- a/js/watch/src/arrivals.test.ts +++ b/js/watch/src/arrivals.test.ts @@ -50,6 +50,21 @@ test("a pending read refuses a concurrent one", async () => { const handle = producer.consume(); const waiting = handle.next(); await expect(handle.next()).rejects.toThrow("multiple calls to next not supported"); + // Woken but not yet resumed: the queued frame still belongs to the pending read. + producer.write(frame(0)); + await expect(handle.next()).rejects.toThrow("multiple calls to next not supported"); + expect(await waiting).toEqual(frame(0)); handle.close(); - expect(await waiting).toBeUndefined(); +}); + +test("leaving a for-await loop closes the handle", async () => { + const producer = new Producer(); + const handle = producer.consume(); + producer.write(frame(0)); + for await (const entry of handle) { + expect(entry).toEqual(frame(0)); + break; + } + producer.write(frame(1)); + expect(await handle.next()).toBeUndefined(); }); diff --git a/js/watch/src/arrivals.ts b/js/watch/src/arrivals.ts index 9684b46788c..6ca03a30ee1 100644 --- a/js/watch/src/arrivals.ts +++ b/js/watch/src/arrivals.ts @@ -27,6 +27,7 @@ interface State { export class Arrivals implements AsyncIterable { readonly #state: State; readonly #open: Set; + #reading = false; /** @internal */ constructor(state: State, open: Set) { @@ -36,25 +37,34 @@ export class Arrivals implements AsyncIterable { /** Get the next received frame, or `undefined` once the handle or its decoder closes. */ async next(): Promise { + if (this.#reading) throw new Error("multiple calls to next not supported"); + this.#reading = true; const state = this.#state; - for (;;) { - const entry = state.queue.shift(); - if (entry) return entry; - if (state.closed) return undefined; - if (state.wake) throw new Error("multiple calls to next not supported"); - await new Promise((resolve) => { - state.wake = resolve; - }); - state.wake = undefined; + try { + for (;;) { + const entry = state.queue.shift(); + if (entry) return entry; + if (state.closed) return undefined; + await new Promise((resolve) => { + state.wake = resolve; + }); + state.wake = undefined; + } + } finally { + this.#reading = false; } } - /** Iterate received frames until the handle or its decoder closes. */ + /** Iterate received frames until the handle or its decoder closes; leaving the loop closes the handle. */ async *[Symbol.asyncIterator](): AsyncIterator { - for (;;) { - const entry = await this.next(); - if (entry === undefined) return; - yield entry; + try { + for (;;) { + const entry = await this.next(); + if (entry === undefined) return; + yield entry; + } + } finally { + this.close(); } } From a0c16a620fc45501d9cc42e7f9532d6df509edb5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 11 Oct 2026 18:29:39 -0700 Subject: [PATCH 08/19] fix(hang): report a latency skip only when content was discarded A delivered group waiting on its done result has nothing unread, so retiring it for the max delay no longer marks the next frame skipped. Co-Authored-By: Claude Opus 5.5 --- js/hang/src/container/consumer.test.ts | 21 +++++++++++++++++++++ js/hang/src/container/consumer.ts | 3 ++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index ef19b1a130b..60ef8263ebf 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -1575,6 +1575,27 @@ test("Consumer latency skip bumps playhead generation once", async () => { consumer.close(); }); +// A delivered group still awaiting its done result holds no unread content, so retiring it +// for the max delay discards nothing. +test("Consumer does not report a skip for a fully delivered group", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 150 as Time.Milli }); + + writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); + await settle(); + expect((await nextFrame(consumer))?.frame?.timestamp).toBe(0 as Time.Micro); + + writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); + await settle(); + + const next = await nextFrame(consumer); + expect(next?.frame?.timestamp).toBe(100_000 as Time.Micro); + expect(next?.skipped).toBe(false); + + consumer.close(); +}); + test("Consumer jumps the playhead after a shed marker with a timestamp hole", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Time.Milli }); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index f4f754810dd..31cf5571bab 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -364,7 +364,8 @@ export class Consumer { if (marker || !ptsContiguous(first.end ?? this.#presentedEnd, nextStart)) { hole = true; } - if (!marker && (!first.empty || !first.done)) this.#skipped = true; + // Only unread or still-arriving content is a skip; a group already delivered discards nothing. + if (!marker && (first.frames.length > 0 || !first.done)) this.#skipped = true; first.consumer.close(); first.frames.length = 0; skipped++; From 26a5c65523c7d261736c1c4705fd6a633aa69c39 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 11 Oct 2026 18:39:07 -0700 Subject: [PATCH 09/19] perf(watch): read arrival backlogs in amortized constant time Advance a head index and compact the read prefix instead of shifting, which can reindex a large backlog on every read in some engines. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/arrivals.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/js/watch/src/arrivals.ts b/js/watch/src/arrivals.ts index 6ca03a30ee1..31f14fe6060 100644 --- a/js/watch/src/arrivals.ts +++ b/js/watch/src/arrivals.ts @@ -14,6 +14,8 @@ export interface Arrival { interface State { queue: Arrival[]; + // The next unread index. Advancing it instead of shifting keeps a long backlog O(1) per read. + head: number; closed: boolean; wake?: () => void; } @@ -42,8 +44,16 @@ export class Arrivals implements AsyncIterable { const state = this.#state; try { for (;;) { - const entry = state.queue.shift(); - if (entry) return entry; + const entry = state.queue[state.head]; + if (entry) { + state.head++; + // Drop the read prefix once it outweighs the rest, which amortizes to O(1). + if (state.head * 2 >= state.queue.length) { + state.queue.splice(0, state.head); + state.head = 0; + } + return entry; + } if (state.closed) return undefined; await new Promise((resolve) => { state.wake = resolve; @@ -72,6 +82,7 @@ export class Arrivals implements AsyncIterable { close(): void { this.#open.delete(this.#state); this.#state.queue.length = 0; + this.#state.head = 0; this.#state.closed = true; this.#state.wake?.(); } @@ -84,7 +95,7 @@ export class Producer { /** Open a handle that receives every later frame. */ consume(): Arrivals { - const state: State = { queue: [], closed: this.#closed }; + const state: State = { queue: [], head: 0, closed: this.#closed }; if (!this.#closed) this.#open.add(state); return new Arrivals(state, this.#open); } From 4a227bf3cc28537ab52591d731e9ef2f0bd095de Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 11 Oct 2026 18:48:33 -0700 Subject: [PATCH 10/19] fix(hang): ignore a queued endpoint marker when detecting a latency skip A legacy video group whose media was delivered but whose endpoint marker is still queued discards no content when the max delay retires it. Co-Authored-By: Claude Opus 5.5 --- js/hang/src/container/consumer.test.ts | 27 ++++++++++++++++++++++++++ js/hang/src/container/consumer.ts | 6 ++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 60ef8263ebf..9c0eaad2979 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -1596,6 +1596,33 @@ test("Consumer does not report a skip for a fully delivered group", async () => consumer.close(); }); +// An unread endpoint marker carries no content, so retiring its group discards nothing. +test("Consumer does not report a skip for a group holding only its endpoint", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("video"), maxDelay: 150 as Time.Milli }); + + const group = new Group.Producer(0); + group.writeFrame({ payload: encodeLegacy(0 as Time.Micro), timestamp: Time.Timestamp.now() }); + group.writeFrame({ + payload: encodeLegacyFrame(33_000 as Time.Micro, new Uint8Array()), + timestamp: Time.Timestamp.now(), + }); + group.close(); + track.writeGroup(group); + await settle(); + expect((await nextFrame(consumer))?.frame?.timestamp).toBe(0 as Time.Micro); + + writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); + await settle(); + + const next = await nextFrame(consumer); + expect(next?.frame?.timestamp).toBe(100_000 as Time.Micro); + expect(next?.skipped).toBe(false); + + consumer.close(); +}); + test("Consumer jumps the playhead after a shed marker with a timestamp hole", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Time.Milli }); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 31cf5571bab..17008a23101 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -364,8 +364,10 @@ export class Consumer { if (marker || !ptsContiguous(first.end ?? this.#presentedEnd, nextStart)) { hole = true; } - // Only unread or still-arriving content is a skip; a group already delivered discards nothing. - if (!marker && (first.frames.length > 0 || !first.done)) this.#skipped = true; + // Only unread or still-arriving media is a skip; a delivered group, or one holding just + // its endpoint marker, discards nothing. + const unread = first.frames.some((frame) => this.#format.end?.(frame) === undefined); + if (!marker && (unread || !first.done)) this.#skipped = true; first.consumer.close(); first.frames.length = 0; skipped++; From 9f07a7a19a9014049c0bf8b3ce049df9c70e8fc7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 12:44:18 -0700 Subject: [PATCH 11/19] refactor(mux): name the read wrapper container::Delivery; split video continuity Rename container::Sample to container::Delivery so it does not collide with PCM or MP4 samples. Complete the Rust continuity quest for container and audio reads and move video propagation to quest/m1/video-continuous.md. Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 2 +- quest/m1/audio-warmup.md | 8 +-- quest/m1/open-gop-leading-pictures.md | 6 +-- quest/m1/rust-continuous.md | 62 ----------------------- quest/m1/video-continuous.md | 39 ++++++++++++++ quest/m2/intra-refresh/consumer-warmup.md | 11 ++-- rs/moq-audio/src/decode/consumer.rs | 32 ++++++------ rs/moq-cli/src/play/media.rs | 2 +- rs/moq-cli/src/play/video.rs | 4 +- rs/moq-ffi/src/audio.rs | 2 +- rs/moq-ffi/src/media.rs | 2 +- rs/moq-mux/src/container/consumer.rs | 14 ++--- rs/moq-mux/src/container/mod.rs | 2 +- 13 files changed, 79 insertions(+), 107 deletions(-) delete mode 100644 quest/m1/rust-continuous.md create mode 100644 quest/m1/video-continuous.md diff --git a/quest/m1/README.md b/quest/m1/README.md index f3ade6dbc7d..409e9398c4b 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -166,7 +166,7 @@ folded into Dropped sources. - [Encoder colour](/quest/m1/color-model.md) - every moq-video encode path signals the colour its output actually has, or refuses instead of mislabelling - [T-STD TS export](/quest/m1/tstd/README.md) - `moq export ts` is a proper remux that passes the T-STD buffer model, starting with a fixed `--delay` - [ATSC AC-3](/quest/m1/ts-atsc-ac3.md) - `export ts` carries 48 kHz ATSC AC-3 at every A/52 rate up to 640 kb/s instead of aborting from 384 kb/s up -- [Rust non-continuous signal](/quest/m1/rust-continuous.md) - the Rust container consumer reports a frame after a subscribe or discontinuity as non-continuous, like JS, for the tune-in and warmup trims +- [Rust video delivery continuity](/quest/m1/video-continuous.md) - Rust video decode reads report each picture's delivery continuity, like the container and audio reads, for the tune-in and warmup trims - [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - a viewer joining at a recovery point drops the leading pictures it cannot decode; continuous viewers keep them - [Audio warmup](/quest/m1/audio-warmup.md) - a viewer joining an Opus rendition mid-stream never hears the unconverged first 80 ms - [#3021](/quest/m1/3021-moq-gst-anchor-generated-media-timelines-to-wall-clock.md) - moq-gst picks the broadcast wall epoch; a restarted source is a new epoch, not a forward re-anchor diff --git a/quest/m1/audio-warmup.md b/quest/m1/audio-warmup.md index 9160f0e82dd..02c92f3a1ac 100644 --- a/quest/m1/audio-warmup.md +++ b/quest/m1/audio-warmup.md @@ -15,8 +15,8 @@ frames decode independently and set nothing; HE-AAC is out of scope. publish `warmup` for Opus; `js/publish` does the same for its Opus track. - `rs/moq-audio/src/decode` already trims Opus `pre_skip` at stream start; the warmup trim is the same mechanism keyed on the container consumer's - non-continuous signal (added in Rust by - [Rust non-continuous signal](/quest/m1/rust-continuous.md)), and the + non-continuous signal (`continuous` on the Rust audio decoder's + `container::Delivery`), and the subscription's maximum age grows by `warmup` as the video consumer quest does. `js/watch` audio mirrors it. - The trim replaces `LEGACY_WARMUP_CALLBACKS` in `js/watch/src/audio/decoder.ts`, @@ -26,10 +26,6 @@ frames decode independently and set nothing; HE-AAC is out of scope. - Tests in both languages: a mid-stream join discards exactly the warmup span and a continuous listener loses nothing. -## Required - -- [Rust non-continuous signal](/quest/m1/rust-continuous.md) - the signal the trim keys on - ## Related - [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - trims video at tune-in from the same signal diff --git a/quest/m1/open-gop-leading-pictures.md b/quest/m1/open-gop-leading-pictures.md index e6fb70d2bee..4ad437f5261 100644 --- a/quest/m1/open-gop-leading-pictures.md +++ b/quest/m1/open-gop-leading-pictures.md @@ -50,8 +50,8 @@ decode, and the JS test should drive the software decoder. Every continuous group is passed through untouched. - The same rule in the Rust decode path (`moq-video` decode consumers), so native playback and the transcoder tune in the same way, keyed on the - non-continuous signal from [Rust non-continuous - signal](/quest/m1/rust-continuous.md). + non-continuous signal from [Rust video delivery + continuity](/quest/m1/video-continuous.md). - The same rule in `moq export ts`. Decided (2026-10-01): the fixed-delay export still sends a join's orphaned leading pictures, so 3 of 500 frames on the open-GOP fixture decode after they present @@ -75,7 +75,7 @@ audio warmup and consumer warmup don't wait on this one. ## Required -- [Rust non-continuous signal](/quest/m1/rust-continuous.md) - the Rust decode path keys the trim on it +- [Rust video delivery continuity](/quest/m1/video-continuous.md) - the Rust decode path keys the trim on it ## Related diff --git a/quest/m1/rust-continuous.md b/quest/m1/rust-continuous.md deleted file mode 100644 index 7bb0604a03f..00000000000 --- a/quest/m1/rust-continuous.md +++ /dev/null @@ -1,62 +0,0 @@ -# [S] Rust non-continuous signal - -## Goal - -`moq_mux::container::Consumer` tells its reader whether a frame picks up -exactly where the previous one left off, like `continuous` in -`js/hang/src/container/consumer.ts`: false on the first frame after the -subscribe and after every discontinuity, true otherwise. -`rs/moq-video/src/decode` and `rs/moq-audio/src/decode` pass it through, so -the open-GOP trim, audio warmup, and consumer warmup key on one signal -instead of each adding its own. - -## Plan - -Split out of [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) -in the 2026-10-08 audit, so audio and consumer warmup need not wait on the -open-GOP trim. - -Today `poll_read` returns a bare frame, and `discontinuity()` is a counter -bumped on a declared marker group, an unproven delivered hole, or a latency -skip, but not on the subscribe itself. Report the flag with each frame and -let the counter go if nothing else needs it. Mirror the JS name. - -Tests: the first frame after a subscribe, a marker group, a delivered hole, -and a latency skip each read as non-continuous, and every other frame as -continuous. - -Public API: breaking in moq-mux (the consumer's read result changes). Wire: -none. - -### Draft progress and remaining video propagation - -The draft adds a shared Sample envelope to container and audio reads, with -regressions for subscribe, markers, holes, skips, and audio decode. Existing -consumers explicitly unwrap the frame; binding surfaces stay unchanged. - -Video still returns its existing decoded Frame. A decoder can return a picture -from before a marker while processing the access unit after it, or return it -only on flush. A single pending boolean attaches the new boundary to the old -picture and loses the new picture boundary, so that approach is not included. -Carry input continuity through codec output association before extending the -video read result. Prefer private decoder metadata association over a -timestamp heuristic, and cover delayed output on both sides of a marker. -The video propagation goal remains open here. Native backends need explicit -input identity, not a map keyed only by presentation time: equal timestamps -can straddle a marker. VideoToolbox and Media Foundation already map synthetic -sample times, OpenH264 queues presentation times, and VAAPI, NVDEC and -MediaCodec carry codec timestamps. Media Foundation also has an unmatched-output -fallback. Decide how a backend reports an output with no surviving input -identity before exposing continuity: recommend an explicit error, rather than -borrowing the latest input's flag. Keep dropped-input bookkeeping bounded and -cover delayed output, duplicate timestamps and decoder recovery on each backend. - -The audio path now drains a resampler's buffered tail before applying a -playhead boundary. A paused-time regression previously returned flags -`false, false, true`, incorrectly joining the old tail into the new span; -it now returns the old body/tail and new body/tail as -`false, true, false, true`. - -## Related - -- [Consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md) - withholds frames after the same signal diff --git a/quest/m1/video-continuous.md b/quest/m1/video-continuous.md new file mode 100644 index 00000000000..2c3b47db74d --- /dev/null +++ b/quest/m1/video-continuous.md @@ -0,0 +1,39 @@ +# [M] Rust video delivery continuity + +## Goal + +`moq_video::decode::Consumer::read` returns `container::Delivery`, so +each decoded picture says whether it continues the previous delivery, like +the container and audio reads already do. A picture decoded from input before +a marker, skip, or subscribe keeps that input's flag even when the codec +returns it late, and the first picture after the boundary reads +non-continuous. + +## Plan + +Split from the container and audio work in +[#5288](https://github.com/moq-dev/moq/pull/5288), where the maintainer chose +the `container::Delivery { frame, continuous }` wrapper and landing video +separately. + +A decoder can return a picture from before a marker while processing the +access unit after it, or only on flush, so a single pending boolean attaches +the boundary to the wrong picture. Carry a private input identity through +every backend (`rs/moq-video/src/decode/backend`) and map each output back to +its input's flag. A map keyed only by presentation time is not enough: equal +timestamps can straddle a marker. VideoToolbox and Media Foundation already +map synthetic sample times, OpenH264 queues presentation times, and VAAPI, +NVDEC and MediaCodec carry codec timestamps. An output whose input identity +did not survive (Media Foundation's unmatched-output fallback) is an explicit +error, never a borrowed flag. Keep dropped-input bookkeeping bounded. + +Tests: delayed output on both sides of a marker, duplicate timestamps, and +decoder recovery, on each backend CI can run. + +Public API: breaking in moq-video (the read result). Bindings keep their +existing frame records. Wire: none. + +## Related + +- [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - the Rust decode trim keys on this signal +- [Consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md) - withholds frames after the same signal diff --git a/quest/m2/intra-refresh/consumer-warmup.md b/quest/m2/intra-refresh/consumer-warmup.md index 536638fcad3..350a6951ae3 100644 --- a/quest/m2/intra-refresh/consumer-warmup.md +++ b/quest/m2/intra-refresh/consumer-warmup.md @@ -22,11 +22,10 @@ replaces both: the rule is timestamp arithmetic on the group start. - Withhold rule, keyed on the same non-continuous signal in both consumers: `js/hang/src/container/consumer.ts` `next()` reports `continuous: false` after a subscribe, a declared discontinuity, or any skip (`#gap`). The Rust - `moq_mux::container::Consumer` gains the equivalent in - [Rust non-continuous signal](/quest/m1/rust-continuous.md) - (today `poll_read` returns a bare frame and only the `discontinuity()` - counter moves); this quest reuses it, and - `rs/moq-video/src/decode/consumer.rs` propagates it. For the first + `moq_mux::container::Consumer` reports the equivalent as + `container::Delivery::continuous`; this quest reuses it once + `rs/moq-video/src/decode/consumer.rs` propagates it in + [Rust video delivery continuity](/quest/m1/video-continuous.md). For the first group after that signal, every frame is decoded (the decoder needs them to build reference state) and frames stamped below `group.start + warmup` are not presented; frames stamped at or above that boundary are, so the recovery @@ -60,7 +59,7 @@ replaces both: the rule is timestamp arithmetic on the group start. ## Required -- [Rust non-continuous signal](/quest/m1/rust-continuous.md) - the Rust signal this keys on +- [Rust video delivery continuity](/quest/m1/video-continuous.md) - the Rust video signal this keys on ## Related diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 908e4f5d773..50abb09d301 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -1,6 +1,6 @@ //! Subscribe to an encoded audio track and emit raw PCM. -use moq_mux::container::Sample; +use moq_mux::container::Delivery; use std::collections::VecDeque; use std::time::{Duration, Instant}; @@ -67,7 +67,7 @@ impl Options { /// /// The mirror of [`encode::Producer`](crate::encode::Producer): output format / /// sample rate / layout are fixed at construction, and -/// [`read`](Self::read) returns samples carrying each frame's delivery continuity +/// [`read`](Self::read) returns deliveries carrying each frame's continuity /// and codec activity. pub struct Consumer { decoder: Decoder, @@ -85,7 +85,7 @@ pub struct Consumer { next_start: Option, /// Frames decoded and not yet handed back, so a gap's tail can be returned /// ahead of the packet that exposed it. - ready: VecDeque>, + ready: VecDeque>, /// Codec activity spans the resampler's buffered output still covers. spans: VecDeque, /// Activity of the last span the output ran past, for the rounding samples the @@ -253,7 +253,7 @@ impl Consumer { } /// Read the next decoded PCM frame, or `None` when the track ends. - /// The returned sample carries the container's delivery continuity through codec buffering. + /// The returned delivery carries the container's continuity through codec buffering. /// /// [`Frame::activity`] reports whether the packet these samples came from /// coded audio. It describes where the frame begins, so a resampled @@ -265,7 +265,7 @@ impl Consumer { /// side stay anchored to their own packet timeline, so the hole is there to /// see. "Doesn't continue" allows for the quantization the stamps carry, which /// on a millisecond-stamped ingest is most of a millisecond. - pub async fn read(&mut self) -> Result>, Error> { + pub async fn read(&mut self) -> Result>, Error> { loop { if let Some(frame) = self.ready.pop_front() { return Ok(Some(frame)); @@ -273,15 +273,15 @@ impl Consumer { let mux_frame = self.track.read().await?; self.apply_discontinuity()?; - let Some(sample) = mux_frame else { + let Some(delivery) = mux_frame else { if let Some(frame) = self.flush()? { let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Sample { frame, continuous }); + self.ready.push_back(Delivery { frame, continuous }); } return Ok(self.ready.pop_front()); }; - let mux_frame = sample.frame; + let mux_frame = delivery.frame; // Every packet the container hands over is an arrival, including one the // decoder is about to refuse, so it is observed before anything can fail. let arrival = jitter::millis(now().duration_since(self.origin)); @@ -311,10 +311,10 @@ impl Consumer { && let Some(frame) = self.gap()? { let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Sample { frame, continuous }); + self.ready.push_back(Delivery { frame, continuous }); } - self.continuous &= sample.continuous; + self.continuous &= delivery.continuous; let rate = self.decoder.sample_rate(); let epoch = *self.epoch.get_or_insert(mux_frame.timestamp); let delay = self.decoder.delay_remaining(); @@ -409,7 +409,7 @@ impl Consumer { // same iteration still comes out first. The next turn of the loop pops it. let frame = self.frame(pcm, timestamp, activity)?; let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Sample { frame, continuous }); + self.ready.push_back(Delivery { frame, continuous }); } } @@ -425,7 +425,7 @@ impl Consumer { // packet declares a new one. Drain before clearing their timeline labels. if let Some(frame) = self.drain()? { let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Sample { frame, continuous }); + self.ready.push_back(Delivery { frame, continuous }); } self.discontinuity = discontinuity; self.next_start = None; @@ -657,8 +657,8 @@ mod tests { } producer.finish().unwrap(); let mut flags = Vec::new(); - while let Some(sample) = consumer.read().await.unwrap() { - flags.push(sample.continuous); + while let Some(delivery) = consumer.read().await.unwrap() { + flags.push(delivery.continuous); } assert_eq!(flags, [false, true, false, true]); } @@ -930,8 +930,8 @@ mod tests { } producer.finish().unwrap(); let mut samples = Vec::new(); - while let Some(sample) = consumer.read().await.unwrap() { - samples.push((sample.frame.timestamp.as_micros(), sample.continuous)); + while let Some(delivery) = consumer.read().await.unwrap() { + samples.push((delivery.frame.timestamp.as_micros(), delivery.continuous)); } assert_eq!( samples.iter().map(|(_, continuous)| *continuous).collect::>(), diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index e0ff402987c..9ba35c89676 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -442,7 +442,7 @@ async fn play_audio( loop { let frame = match consumer.read().await { - Ok(Some(sample)) => sample.frame, + Ok(Some(delivery)) => delivery.frame, Ok(None) => break, // One bad packet is that packet's problem: the decoder stays usable, so // skip it rather than ending playback and taking the video window down diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index e42366f8c60..a97506f3600 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -84,8 +84,8 @@ impl Video { self.presentation.lock().unwrap().video_restarted(); } let ended = frame.is_none(); - if let Some(sample) = frame { - let frame = sample.frame; + if let Some(delivery) = frame { + let frame = delivery.frame; if self .presentation .lock() diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index 0d8d1ba3820..5d3463a85a8 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -342,7 +342,7 @@ struct ConsumerInner { impl ConsumerInner { async fn next(&mut self) -> Result, MoqError> { - Ok(self.consumer.read().await?.map(|sample| sample.frame.into())) + Ok(self.consumer.read().await?.map(|delivery| delivery.frame.into())) } } diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 88f5ee0e448..0c8fc417b17 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -1104,7 +1104,7 @@ impl Media { self.inner .read() .await? - .map(|sample| media_frame(sample.frame)) + .map(|delivery| media_frame(delivery.frame)) .transpose() } } diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 51232cdd91f..6959ca36696 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -3,7 +3,7 @@ use std::task::{Poll, ready}; use moq_net::Timestamp; -use super::{Container, Frame, Sample, TimestampRewind}; +use super::{Container, Delivery, Frame, TimestampRewind}; /// Media and clean group boundaries in delivery order. pub(crate) enum Event { @@ -179,9 +179,9 @@ impl Consumer { /// configured max delay. /// /// Returns `None` when the track has ended. - /// The first sample and the first one after a playhead event have `continuous: false`. + /// The first delivery and the first one after a playhead event have `continuous: false`. /// An undeclared forward timestamp jump alone does not change the flag. - pub async fn read(&mut self) -> Result, F::Error> { + pub async fn read(&mut self) -> Result, F::Error> { kio::wait(|waiter| self.poll_read(waiter)).await } @@ -189,13 +189,13 @@ impl Consumer { /// /// Uses a single waiter that gets registered on all relevant kio channels, /// avoiding the need for `tokio::select!` or `FuturesUnordered`. - pub fn poll_read(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { + pub fn poll_read(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { loop { match ready!(self.poll_event(waiter))? { Some(Event::Frame(frame)) => { let continuous = self.delivered == Some(self.discontinuity); self.delivered = Some(self.discontinuity); - return Poll::Ready(Ok(Some(Sample { frame, continuous }))); + return Poll::Ready(Ok(Some(Delivery { frame, continuous }))); } Some(Event::FrameEnd(end)) => { if self.format.kind() == super::Kind::Audio { @@ -928,8 +928,8 @@ mod tests { write_group(&mut track, 4, &[ts(2_000_000), ts(2_010_000)]); track.finish().unwrap(); let mut flags = Vec::new(); - while let Some(sample) = reader.read().await.unwrap() { - flags.push(sample.continuous); + while let Some(delivery) = reader.read().await.unwrap() { + flags.push(delivery.continuous); } assert_eq!(flags, [false, true, false, true, false, true]); } diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 76890fd8a33..77670d47115 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -47,7 +47,7 @@ pub enum Kind { /// A delivered frame and whether it continues the previous delivery. #[derive(Clone, Debug)] -pub struct Sample { +pub struct Delivery { /// The encoded or decoded media frame. pub frame: T, /// False on the first frame and after a playhead discontinuity. From 5a0e640119e68fc0fe29dcba4bbaf9288728192a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 13:16:13 -0700 Subject: [PATCH 12/19] fix(mux): mark the delivery after an evicted current group non-continuous An aborted current group skipped to the next buffered group without advancing the playhead, so the next frame read as continuous although the rest of the evicted group was lost. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/consumer.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 6959ca36696..42f4703bc86 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -53,8 +53,9 @@ pub(crate) enum Event { /// /// A marker group (no decodable frames, one empty payload) is a walk-now discontinuity. /// A delivered sequence hole is a playhead event unless the boundary is contiguous within -/// 1 ms, and a latency skip is the same event. [`discontinuity`](Self::discontinuity) is -/// that playhead generation: re-apply startup delay and skip, not a decoder flush. +/// 1 ms, and a latency skip or an evicted current group is the same event. +/// [`discontinuity`](Self::discontinuity) is that playhead generation: re-apply startup +/// delay and skip, not a decoder flush. /// Empty groups (zero objects) mean nothing. A group whose timestamps fall below the live /// edge earlier groups reached is malformed and aborts the track. pub struct Consumer { @@ -151,7 +152,7 @@ impl Consumer { } /// A counter that increments at each playhead event: a declared marker group, an - /// unproven delivered hole, or a latency skip. + /// unproven delivered hole, a latency skip, or an evicted current group. /// /// Downstream consumers re-apply startup delay and skip when it changes. It is not a /// decoder flush: the next group already starts on a keyframe with parameter sets. @@ -307,6 +308,8 @@ impl Consumer { error = ?e, "current group evicted; skipping to next buffered group" ); + // Whatever the group still held is lost, so this is a skip like any other. + self.bump_playhead(); self.pending.pop_front(); self.current = self.pending.front().map_or(self.current + 1, |g| g.sequence); continue 'read; @@ -1874,6 +1877,7 @@ mod tests { ts(150_000), "skipped the evicted gap to the live group" ); + assert!(!next.continuous, "the eviction skip starts a new span"); } /// A missing (evicted) sequence with a newer group buffered must be skipped once the From 48ec8c687ced34f055b0103531b9178b66be871f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 19:42:21 -0700 Subject: [PATCH 13/19] feat(mux): read gaps and declared ends as items in delivery order Replace the container::Delivery wrapper and the polled discontinuity()/end() accessors with one read stream in both languages: - Rust: container::Read::{Frame { group, frame }, Gap, End { group, end }} - JS: Consumer.next() yields { kind: "frame", group, frame } | { kind: "gap" } | { kind: "end", group, end }; the continuous/discontinuity fields, the group-done results, and the discontinuity getter are gone. A gap is media this reader will never get (the join, a skip, an eviction, a truncation, an unproven hole). An end is the publisher declaring its media stops: a marker group or an audio terminal. The frames that follow an end from its own group are the stopped encoder's drain; the next group is fresh. Timestamps cannot tell those apart (drain packets are stamped past the end), so the group rides along, as the hang draft scopes the endpoint. Trailing video duration markers stay internal. moq-cli play now applies a gap or end before the frame after it instead of polling a counter after the read. Drops quest/m1/video-continuous.md: decoders read the gap directly. Co-Authored-By: Claude Opus 5.5 --- doc/lib/rs/moq-mux.md | 6 +- .../container/consumer.nonsequential.test.ts | 15 +- .../src/container/consumer.outoforder.test.ts | 53 +-- js/hang/src/container/consumer.rewind.test.ts | 3 +- js/hang/src/container/consumer.skips.test.ts | 3 +- .../src/container/consumer.startup.test.ts | 66 ++-- js/hang/src/container/consumer.test.ts | 283 ++++++-------- js/hang/src/container/consumer.ts | 147 +++---- js/hang/src/container/index.ts | 2 +- js/watch/src/audio/decoder.test.ts | 14 +- js/watch/src/audio/decoder.ts | 28 +- js/watch/src/audio/playout.test.ts | 2 +- js/watch/src/audio/replay.ts | 4 +- js/watch/src/audio/terminal.test.ts | 51 ++- js/watch/src/audio/terminal.ts | 53 +-- js/watch/src/retention.test.ts | 6 +- js/watch/src/video/decoder.test.ts | 46 +-- js/watch/src/video/decoder.ts | 66 ++-- quest/m1/README.md | 1 - quest/m1/audio-warmup.md | 6 +- quest/m1/open-gop-leading-pictures.md | 27 +- quest/m1/video-continuous.md | 39 -- quest/m2/intra-refresh/consumer-warmup.md | 18 +- rs/hang/examples/subscribe.rs | 18 +- rs/moq-audio/src/decode/consumer.rs | 244 +++++------- rs/moq-audio/src/encode/capture.rs | 7 +- rs/moq-audio/src/encode/producer.rs | 53 +-- rs/moq-audio/src/playback.rs | 2 +- rs/moq-audio/tests/import.rs | 2 +- rs/moq-audio/tests/roundtrip.rs | 7 +- rs/moq-c/src/audio.rs | 2 +- rs/moq-c/src/consume.rs | 8 +- rs/moq-cli/src/play/media.rs | 2 +- rs/moq-cli/src/play/video.rs | 47 +-- rs/moq-cli/src/publish.rs | 29 +- rs/moq-ffi/src/audio.rs | 2 +- rs/moq-ffi/src/media.rs | 13 +- rs/moq-gst/src/source/imp.rs | 5 +- rs/moq-mux/src/codec/opus/import.rs | 6 +- rs/moq-mux/src/codec/vp8/import.rs | 6 +- rs/moq-mux/src/container/consumer.rs | 363 +++++++++++------- rs/moq-mux/src/container/flv/import_test.rs | 30 +- rs/moq-mux/src/container/fmp4/export.rs | 2 +- rs/moq-mux/src/container/fmp4/import_test.rs | 6 +- rs/moq-mux/src/container/group.rs | 10 +- rs/moq-mux/src/container/mkv/import_test.rs | 14 +- rs/moq-mux/src/container/mod.rs | 27 +- rs/moq-mux/src/container/producer.rs | 4 +- rs/moq-mux/src/container/source.rs | 2 +- rs/moq-mux/src/container/test_util.rs | 4 +- rs/moq-mux/src/container/ts/export_test.rs | 16 +- rs/moq-mux/src/container/ts/import.rs | 41 +- rs/moq-mux/src/container/ts/import_test.rs | 25 +- rs/moq-mux/src/container/ts/programs.rs | 13 +- rs/moq-mux/src/import/track.rs | 14 +- rs/moq-rtc/src/codec/mod.rs | 15 +- rs/moq-srt/src/ts.rs | 25 +- rs/moq-video/src/capture/wgc.rs | 8 +- rs/moq-video/src/decode/consumer.rs | 25 +- rs/moq-video/src/decode/decoder.rs | 11 +- rs/moq-video/src/encode/producer.rs | 8 +- 61 files changed, 1028 insertions(+), 1027 deletions(-) delete mode 100644 quest/m1/video-continuous.md diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index de9782a4f65..82255ac22ce 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -30,9 +30,9 @@ set at the init segment, so a new rendition or a changed configuration ends the export. The [CLI page](/bin/cli#import) covers these and the MPEG-TS specifics: one program per broadcast, damaged packets, and feed checks. -Container and audio reads return a frame with its delivery continuity: the first -frame and the first after a skip, hole, or declared break start a new span. -Video decoders return bare frames. +A container read yields frames plus two markers in delivery order: a gap where +media will never arrive (the join, a skip, a hole) and an end where the +publisher declared its media stops (a pause, a source switch, the finish). Data tracks are catalog entries too. The [hang page](/concept/hang#data-tracks) describes the modes. A capture time converts onto the broadcast clock; a diff --git a/js/hang/src/container/consumer.nonsequential.test.ts b/js/hang/src/container/consumer.nonsequential.test.ts index fca4c3f4a76..61a62ea97a9 100644 --- a/js/hang/src/container/consumer.nonsequential.test.ts +++ b/js/hang/src/container/consumer.nonsequential.test.ts @@ -5,6 +5,14 @@ import { Format as LegacyFormat } from "./legacy.ts"; // Standalone from consumer.test.ts so it only pulls the Legacy format, not the CMAF decoder. +/** The next frame, passing over gaps and ends. */ +async function nextFrame(consumer: Consumer) { + for (;;) { + const next = await consumer.next(); + if (!next || next.kind === "frame") return next; + } +} + function encodeLegacyFrame(timestamp: Time.Micro, payload: Uint8Array): Uint8Array { const tsBytes = Varint.encode(timestamp); const data = new Uint8Array(tsBytes.byteLength + payload.byteLength); @@ -40,10 +48,9 @@ test("Consumer waits on a non-sequential hang group because it has no per-frame a.close(); track.writeGroup(a); - // Drain A: its frame, then its group-done marker. - const firstFrame = await consumer.next(); - expect(firstFrame?.frame?.payload).toEqual(new Uint8Array([0x01])); - await consumer.next(); + // Drain A's frame. + const firstFrame = await nextFrame(consumer); + expect(firstFrame?.frame.payload).toEqual(new Uint8Array([0x01])); // Park next() before the next group's frames arrive (the live streaming case). const pending = consumer.next(); diff --git a/js/hang/src/container/consumer.outoforder.test.ts b/js/hang/src/container/consumer.outoforder.test.ts index 89453f3d1f6..2bccec79ca7 100644 --- a/js/hang/src/container/consumer.outoforder.test.ts +++ b/js/hang/src/container/consumer.outoforder.test.ts @@ -5,6 +5,14 @@ import { Format as LegacyFormat } from "./legacy.ts"; // Standalone from consumer.test.ts so it only pulls the Legacy format, not the CMAF decoder. +/** The next frame, passing over gaps and ends. */ +async function nextFrame(consumer: Consumer) { + for (;;) { + const next = await consumer.next(); + if (!next || next.kind === "frame") return next; + } +} + function encodeLegacyFrame(timestamp: Time.Micro, payload: Uint8Array): Uint8Array { const tsBytes = Varint.encode(timestamp); const data = new Uint8Array(tsBytes.byteLength + payload.byteLength); @@ -29,14 +37,16 @@ function publish(track: Track.Producer, sequence: number, millis: number) { group.close(); } -/** Every group and frame the consumer will hand over right now, as [group, payload] pairs. */ +/** Every frame the consumer will hand over right now, as [group, payload] pairs. */ async function drain(consumer: Consumer): Promise<[number, number | undefined][]> { const seen: [number, number | undefined][] = []; for (;;) { - const next = await Promise.race([consumer.next(), new Promise((resolve) => setTimeout(resolve, 50))]); + const next = await Promise.race([ + consumer.next(), + new Promise((resolve) => setTimeout(resolve, 50)), + ]); if (!next) break; - const { group, frame } = next as { group: number; frame?: { payload: Uint8Array } }; - seen.push([group, frame?.payload[0]]); + if (next.kind === "frame") seen.push([next.group, next.frame.payload[0]]); } return seen; } @@ -55,7 +65,7 @@ test("out-of-order groups are delivered rather than dropped", async () => { // The live edge lands first, and delivery starts there rather than waiting. publish(track, 3, 30); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([3])); + expect((await nextFrame(consumer))?.frame.payload).toEqual(new Uint8Array([3])); // The window's head follows it. Every group is still within both budgets, so all three // are handed over even though each sits below the group already delivered. @@ -63,8 +73,7 @@ test("out-of-order groups are delivered rather than dropped", async () => { publish(track, 1, 10); publish(track, 2, 20); - const delivered = (await drain(consumer)).filter(([, payload]) => payload !== undefined); - expect(delivered).toEqual([ + expect(await drain(consumer)).toEqual([ [0, 0], [1, 1], [2, 2], @@ -87,25 +96,25 @@ test("a below-cursor group still downloading is not truncated", async () => { const live = new Group.Producer(3); track.writeGroup(live); publishFrame(live, 3000, 30); - expect((await consumer.next())?.frame?.payload[0]).toBe(30); + expect((await nextFrame(consumer))?.frame.payload[0]).toBe(30); // A backlog group lands below the cursor, one frame at a time (mid-download). const backlog = new Group.Producer(1); track.writeGroup(backlog); publishFrame(backlog, 1000, 10); - const first = await consumer.next(); + const first = await nextFrame(consumer); expect(first?.group).toBe(1); - expect(first?.frame?.payload[0]).toBe(10); + expect(first?.frame.payload[0]).toBe(10); // The consumer finds the group's buffer empty while it is still open. It must wait // rather than shift the group out, so the second frame still arrives. - const pending = consumer.next(); + const pending = nextFrame(consumer); publishFrame(backlog, 1500, 11); backlog.close(); - const second = (await pending) as { group: number; frame?: { payload: Uint8Array } }; - expect(second.group).toBe(1); - expect(second.frame?.payload[0]).toBe(11); + const second = await pending; + expect(second?.group).toBe(1); + expect(second?.frame.payload[0]).toBe(11); live.close(); track.close(); @@ -120,30 +129,24 @@ test("delayed older groups survive a floor established by two newer groups", asy format: new LegacyFormat("data"), maxDelay: Time.Milli(5000), }); - async function nextFrame() { - for (;;) { - const next = await consumer.next(); - if (!next || next.frame) return next; - } - } try { publish(track, 3, 30); - expect((await nextFrame())?.group).toBe(3); + expect((await nextFrame(consumer))?.group).toBe(3); const latest = new Group.Producer(4); track.writeGroup(latest); publishFrame(latest, 40, 4); - expect((await nextFrame())?.group).toBe(4); + expect((await nextFrame(consumer))?.group).toBe(4); // Group 1's 10ms frame is valid history, below group 4's 30ms floor. publish(track, 1, 10); - const backlog = await nextFrame(); + const backlog = await nextFrame(consumer); expect(backlog?.group).toBe(1); - expect(backlog?.frame?.timestamp).toBe(Time.Micro(10_000)); + expect(backlog?.frame.timestamp).toBe(Time.Micro(10_000)); // Delivering the backlog must not erase the floor for group 4 itself. publishFrame(latest, 29, 5); - await expect(nextFrame()).rejects.toThrow("below the previous group start"); + await expect(nextFrame(consumer)).rejects.toThrow("below the previous group start"); } finally { consumer.close(); track.close(); diff --git a/js/hang/src/container/consumer.rewind.test.ts b/js/hang/src/container/consumer.rewind.test.ts index c826fbe1b58..34adadab02e 100644 --- a/js/hang/src/container/consumer.rewind.test.ts +++ b/js/hang/src/container/consumer.rewind.test.ts @@ -38,7 +38,7 @@ function group(track: Track.Producer, timestamps: number[], end?: number) { async function frame(consumer: Consumer) { for (;;) { const result = await consumer.next(); - if (!result || result.frame) return result; + if (!result || result.kind === "frame") return result; } } @@ -50,7 +50,6 @@ test("Consumer accepts a resuming keyframe before the estimated group end", asyn group(track, [0, 33_000], 66_000); expect((await frame(consumer))?.frame?.timestamp).toBe(Time.Micro(0)); expect((await frame(consumer))?.frame?.timestamp).toBe(Time.Micro(33_000)); - expect((await consumer.next())?.end).toBe(Time.Micro(66_000)); group(track, [50_000]); expect((await frame(consumer))?.frame?.timestamp).toBe(Time.Micro(50_000)); } finally { diff --git a/js/hang/src/container/consumer.skips.test.ts b/js/hang/src/container/consumer.skips.test.ts index 591ab00aa1b..17447495cf3 100644 --- a/js/hang/src/container/consumer.skips.test.ts +++ b/js/hang/src/container/consumer.skips.test.ts @@ -34,7 +34,8 @@ for (const count of [1, 16, 256]) { expect(warn).toHaveBeenLastCalledWith(`skipping slow groups: track=tone 0 -> ${count * 2} count=${count}`); expect(consumer.buffered.peek()[0]?.start).toBe(Time.Milli(count)); // The retained group is still delivered, and another catch-up reports separately. - expect((await consumer.next())?.group).toBe(count * 2); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: count * 2 }); newest.writeFrame({ payload: Varint.encode(count * 1000), timestamp: Time.Timestamp.now() }); write(count * 2 + 2, (count + 1) * 1000); await buffered(count + 1); diff --git a/js/hang/src/container/consumer.startup.test.ts b/js/hang/src/container/consumer.startup.test.ts index 008116a911f..5d8ecfacd8f 100644 --- a/js/hang/src/container/consumer.startup.test.ts +++ b/js/hang/src/container/consumer.startup.test.ts @@ -1,6 +1,6 @@ import { expect, jest, onTestFinished, spyOn, test } from "bun:test"; import { Group, Error as NetError, StreamCode, Time, Track } from "@moq/net"; -import { Consumer } from "./consumer.ts"; +import { Consumer, type Read } from "./consumer.ts"; import { encodeFrame, Format } from "./legacy.ts"; // Drain the in-memory readers and their continuations without advancing mocked time. @@ -13,7 +13,7 @@ function setup(maxDelay = 100) { format: new Format("video"), maxDelay: Time.Milli(maxDelay), }); - const results: NonNullable>>[] = []; + const results: Read[] = []; const warn = spyOn(console, "warn").mockImplementation(() => {}); const reading = (async () => { for (;;) { @@ -35,6 +35,19 @@ function setup(maxDelay = 100) { return { track, consumer, results, warn }; } +// Each item as "gap", "@<µs>" for a frame, or "end @<µs>". +function items(results: Read[]): string[] { + return results.map((read) => { + if (read.kind === "gap") return "gap"; + if (read.kind === "end") return `end ${read.group}@${read.end}`; + return `${read.group}@${read.frame.timestamp}`; + }); +} + +function frames(results: Read[]) { + return results.flatMap((read) => (read.kind === "frame" ? [read] : [])); +} + function write(group: Group.Producer, timestamp: number, marker = false) { group.writeFrame({ payload: encodeFrame(marker ? new Uint8Array() : new Uint8Array([1]), Time.Micro(timestamp)), @@ -46,7 +59,7 @@ for (const emptySuccessor of [false, true]) { test.each([0, 100])( `Consumer starts across a hole after an empty reset (empty successor: ${emptySuccessor}, maxDelay: %s)`, async (maxDelay) => { - const { track, consumer, results, warn } = setup(maxDelay); + const { track, results, warn } = setup(maxDelay); const stale = new Group.Producer(16); track.writeGroup(stale); await flush(); @@ -58,8 +71,7 @@ for (const emptySuccessor of [false, true]) { } stale.close(new NetError.Stream(StreamCode.Old)); await flush(); - expect(results.some((result) => result.group === 16 && !result.frame)).toBe(true); - if (emptySuccessor) expect(results.some((result) => result.group === 17 && !result.frame)).toBe(true); + expect(results).toEqual([]); const sequence = emptySuccessor ? 19 : 18; const live = new Group.Producer(sequence); @@ -67,36 +79,34 @@ for (const emptySuccessor of [false, true]) { for (let i = 0; i < 5; i++) write(live, 1_000_000 + i * 33_333); await flush(); - const frames = results.filter((result) => result.frame); - expect(frames.map((result) => result.group)).toEqual(Array(5).fill(sequence)); - expect(frames.map((result) => result.frame?.timestamp)).toEqual( - [1_000_000, 1_033_333, 1_066_666, 1_099_999, 1_133_332].map((timestamp) => Time.Micro(timestamp)), - ); - expect(frames.map((result) => result.frame?.keyframe)).toEqual([true, false, false, false, false]); - expect(frames.map((result) => result.continuous)).toEqual([false, true, true, true, true]); - expect(consumer.discontinuity).toBe(0); + expect(items(results)).toEqual([ + "gap", + ...[1_000_000, 1_033_333, 1_066_666, 1_099_999, 1_133_332].map( + (timestamp) => `${sequence}@${timestamp}`, + ), + ]); + expect(frames(results).map((read) => read.frame.keyframe)).toEqual([true, false, false, false, false]); expect(warn).not.toHaveBeenCalled(); }, ); } test("Consumer preserves a declared marker before its first media across a hole", async () => { - const { track, consumer, results, warn } = setup(); + const { track, results, warn } = setup(); const marker = new Group.Producer(16); track.writeGroup(marker); write(marker, 0, true); marker.close(); await flush(); - expect(consumer.discontinuity).toBe(1); + expect(items(results)).toEqual(["gap", "end 16@0"]); const live = new Group.Producer(18); track.writeGroup(live); write(live, 50_000); await flush(); - const frames = results.filter((result) => result.frame); - expect(frames).toHaveLength(1); - expect(frames[0]).toMatchObject({ group: 18, discontinuity: 1, continuous: false }); - expect(frames[0].frame?.keyframe).toBe(true); + // Group 17 never arrived, so the media after the marker is past a gap. + expect(items(results)).toEqual(["gap", "end 16@0", "gap", "18@50000"]); + expect(frames(results)[0].frame.keyframe).toBe(true); expect(warn).not.toHaveBeenCalled(); }); @@ -107,7 +117,7 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(first, 0); first.close(); await flush(); - expect(results.filter((result) => result.frame)).toHaveLength(1); + expect(items(results)).toEqual(["gap", "16@0"]); if (reset) { const marker = new Group.Producer(17); @@ -115,7 +125,7 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(marker, 0, true); marker.close(); await flush(); - expect(consumer.discontinuity).toBe(1); + expect(items(results)).toEqual(["gap", "16@0", "end 17@0"]); } const sequence = reset ? 20 : 19; const later = new Group.Producer(sequence); @@ -123,7 +133,7 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(later, 50_000); await flush(); expect(consumer.buffered.peek().length).toBeGreaterThan(0); - expect(results.filter((result) => result.frame)).toHaveLength(1); + expect(frames(results)).toHaveLength(1); // Supply the missing, timestamp-contiguous bridge and prove the queued frame survives. const bridge = new Group.Producer(sequence - 1); @@ -131,11 +141,13 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(bridge, 0); bridge.close(); await flush(); - expect(results.filter((result) => result.frame).map((result) => result.group)).toEqual([ - 16, - sequence - 1, - sequence, + // The bridge closes the hole, so nothing reads as a gap past the first. + expect(items(results)).toEqual([ + "gap", + "16@0", + ...(reset ? ["end 17@0"] : []), + `${sequence - 1}@0`, + `${sequence}@50000`, ]); - expect(consumer.discontinuity).toBe(reset ? 1 : 0); expect(warn).not.toHaveBeenCalled(); }); diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 0443a095d38..a2c740a4002 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -5,7 +5,7 @@ import { AudioConfigSchema } from "../catalog/audio.ts"; import { decodeInitSegment, type InitSegment } from "./cmaf/decode.ts"; import { createAudioInitSegment, encodeDataSegment } from "./cmaf/encode.ts"; import { Format as CmafFormat } from "./cmaf/format.ts"; -import { Consumer } from "./consumer.ts"; +import { Consumer, type Read } from "./consumer.ts"; import type { Format as ContainerFormat } from "./format.ts"; import { Format as LegacyFormat, Producer as LegacyProducer } from "./legacy.ts"; import type { Frame } from "./types.ts"; @@ -522,13 +522,27 @@ async function drainFrames( new Promise((resolve) => setTimeout(() => resolve(null), timeout)), ]); if (result === null || result === undefined) break; - if (result.frame) { + if (result.kind === "frame") { frames.push({ timestamp: result.frame.timestamp, group: result.group, keyframe: result.frame.keyframe }); } } return frames; } +/** An item as "gap", a frame's µs, or "end <µs>". */ +function describe(read: Read): string { + if (read.kind === "gap") return "gap"; + if (read.kind === "end") return `end ${read.end}`; + return `${read.frame.timestamp}`; +} + +/** The frame a read carries, failing on anything else. */ +function frameOf(read: Read | string | undefined): Frame { + if (typeof read !== "object" || read.kind !== "frame") + throw new Error(`expected a frame, got ${JSON.stringify(read)}`); + return read.frame; +} + test("Consumer delivers frames from a single group", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Time.Milli }); @@ -731,7 +745,7 @@ test("Consumer delivers a group that arrives below the cursor", async () => { // --- Group.Producer boundary signals --- -test("Consumer next() returns group-done signals", async () => { +test("Consumer next() yields a gap before the first frame only", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Time.Milli }); @@ -741,26 +755,21 @@ test("Consumer next() returns group-done signals", async () => { await new Promise((resolve) => setTimeout(resolve, 50)); - const allResults: { frame: boolean; group: number }[] = []; + const seen: string[] = []; for (;;) { const result = await Promise.race([ consumer.next(), new Promise((resolve) => setTimeout(() => resolve(null), 500)), ]); if (result === null || result === undefined) break; - allResults.push({ frame: result.frame !== undefined, group: result.group }); + seen.push(describe(result)); } - const frameResults = allResults.filter((r) => r.frame); - const boundaries = allResults.filter((r) => !r.frame); - expect(frameResults).toHaveLength(3); - expect(boundaries).toHaveLength(2); - expect(boundaries[0].group).toBe(0); - expect(boundaries[1].group).toBe(1); + expect(seen).toEqual(["gap", "0", "33000", "66000"]); consumer.close(); }); -test("Consumer reports a duration marker as metadata", async () => { +test("Consumer keeps a trailing duration marker internal", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("video"), @@ -781,18 +790,17 @@ test("Consumer reports a duration marker as metadata", async () => { track.close(); await settle(); - const media = await consumer.next(); - expect(media?.frame?.payload).toEqual(new Uint8Array([0xde, 0xad])); - expect(media?.frame?.keyframe).toBe(true); - expect(media?.frame?.duration).toBeUndefined(); - - const done = await consumer.next(); - expect(done?.frame).toBeUndefined(); - expect(done?.end).toBe(33_000 as Time.Micro); + expect(await consumer.next()).toEqual({ kind: "gap" }); + const media = frameOf(await consumer.next()); + expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); + expect(media.keyframe).toBe(true); + expect(media.duration).toBeUndefined(); + // The marker is the frame's duration, not an item. + expect(await consumer.next()).toBeUndefined(); consumer.close(); }); -test("Consumer skips a leading marker and keeps the first media keyframe", async () => { +test("Consumer reads a leading marker as an end and keeps the first media keyframe", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("video"), @@ -812,17 +820,16 @@ test("Consumer skips a leading marker and keeps the first media keyframe", async track.writeGroup(group); track.close(); - const marker = await consumer.next(); - expect(marker?.end).toBe(20_000 as Time.Micro); - expect(marker?.frame).toBeUndefined(); - const media = await consumer.next(); - expect(media?.frame?.payload).toEqual(new Uint8Array([0xde, 0xad])); - expect(media?.frame?.keyframe).toBe(true); - expect(media?.frame?.duration).toBeUndefined(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(await consumer.next()).toEqual({ kind: "end", group: 0, end: 20_000 as Time.Micro }); + const media = frameOf(await consumer.next()); + expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); + expect(media.keyframe).toBe(true); + expect(media.duration).toBeUndefined(); consumer.close(); }); -test("Consumer skips an empty LOC payload", async () => { +test("Consumer reads an empty LOC payload as the frame's duration", async () => { const track = new Track.Producer("test"); const producer = new LocProducer(track); producer.encode(new Uint8Array([0xde, 0xad]), 0 as Time.Micro, true); @@ -831,16 +838,15 @@ test("Consumer skips an empty LOC payload", async () => { const consumer = new Consumer(replay(track), { format: new LocFormat("video"), maxDelay: 500 as Time.Milli }); await settle(); - const media = await consumer.next(); - expect(media?.frame?.payload).toEqual(new Uint8Array([0xde, 0xad])); - expect(media?.frame?.duration).toBeUndefined(); - const marker = await consumer.next(); - expect(marker?.end).toBe(33_000 as Time.Micro); - expect(marker?.frame).toBeUndefined(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + const media = frameOf(await consumer.next()); + expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); + expect(media.duration).toBeUndefined(); + expect(await consumer.next()).toBeUndefined(); consumer.close(); }); -test("Consumer skips the duration marker the LOC producer writes", async () => { +test("Consumer passes over the duration markers the LOC producer writes", async () => { const track = new Track.Producer("test"); const producer = new LocProducer(track); producer.encode(new Uint8Array([0xde, 0xad]), 0 as Time.Micro, true); @@ -850,23 +856,15 @@ test("Consumer skips the duration marker the LOC producer writes", async () => { const consumer = new Consumer(replay(track), { format: new LocFormat("video"), maxDelay: 500 as Time.Milli }); await settle(); - const first = await consumer.next(); - expect(first?.frame?.payload).toEqual(new Uint8Array([0xde, 0xad])); - expect(first?.frame?.keyframe).toBe(true); - const second = await consumer.next(); - expect(second?.frame?.payload).toEqual(new Uint8Array([0xbe, 0xef])); - const boundary = await consumer.next(); - expect(boundary?.frame).toBeUndefined(); - expect(boundary?.end).toBe(20_000 as Time.Micro); - const keyframe = await consumer.next(); - expect(keyframe?.frame).toBeUndefined(); - expect(keyframe?.end).toBeUndefined(); - const next = await consumer.next(); - expect(next?.frame?.payload).toEqual(new Uint8Array([0xca, 0xfe])); - expect(next?.frame?.keyframe).toBe(true); - const tail = await consumer.next(); - expect(tail?.frame).toBeUndefined(); - expect(tail?.end).toBe(30_000 as Time.Micro); + expect(await consumer.next()).toEqual({ kind: "gap" }); + const first = frameOf(await consumer.next()); + expect(first.payload).toEqual(new Uint8Array([0xde, 0xad])); + expect(first.keyframe).toBe(true); + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0xbe, 0xef])); + const next = frameOf(await consumer.next()); + expect(next.payload).toEqual(new Uint8Array([0xca, 0xfe])); + expect(next.keyframe).toBe(true); + expect(await consumer.next()).toBeUndefined(); consumer.close(); }); @@ -879,21 +877,21 @@ test("Consumer preserves empty Legacy and LOC data frames", async () => { producer.close(); const format = kind === "legacy" ? new LegacyFormat("data") : new LocFormat(); const consumer = new Consumer(replay(track), { format, maxDelay: 500 as Time.Milli }); - const result = await consumer.next(); - expect(result?.frame?.payload).toEqual(new Uint8Array()); - expect(result?.frame?.keyframe).toBe(true); - expect(result?.end).toBeUndefined(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + const frame = frameOf(await consumer.next()); + expect(frame.payload).toEqual(new Uint8Array()); + expect(frame.keyframe).toBe(true); consumer.close(); } }); // --- Malformed rewind --- -/** Read until the next media frame, skipping the group-done markers in between. */ +/** Read until the next media frame, passing over gaps and ends. */ async function nextFrame(consumer: Consumer) { for (;;) { const result = await consumer.next(); - if (!result || result.frame) return result; + if (!result || result.kind === "frame") return result; } } @@ -918,7 +916,6 @@ test("Consumer aborts a rewind carried by a later arrival", async () => { writeGroupWithLegacyFrames(track, 0, [10_000_000 as Time.Micro]); await settle(); expect((await nextFrame(consumer))?.frame?.timestamp).toBe(10_000_000 as Time.Micro); - expect((await consumer.next())?.frame).toBeUndefined(); const pending = nextFrame(consumer); await settle(); @@ -937,20 +934,17 @@ test("Consumer treats B-frame reordering within a group as continuous", async () writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 66_000 as Time.Micro, 33_000 as Time.Micro]); await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); for (const timestamp of [0, 66_000, 33_000]) { - const result = await nextFrame(consumer); - expect(result?.frame?.timestamp).toBe(timestamp as Time.Micro); - expect(result?.discontinuity).toBe(0); + expect(frameOf(await consumer.next()).timestamp).toBe(timestamp as Time.Micro); } // The cursor is now on group 1, whose own frames reorder the same way. writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro, 166_000 as Time.Micro, 133_000 as Time.Micro]); await settle(); for (const timestamp of [100_000, 166_000, 133_000]) { - const result = await nextFrame(consumer); - expect(result?.frame?.timestamp).toBe(timestamp as Time.Micro); - expect(result?.discontinuity).toBe(0); - expect(result?.continuous).toBe(true); + // Read directly: a gap here would fail the frame check. + expect(frameOf(await consumer.next()).timestamp).toBe(timestamp as Time.Micro); } consumer.close(); @@ -963,10 +957,9 @@ test("Consumer accepts open-GOP leading pictures above the previous group", asyn writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 33_000 as Time.Micro]); writeGroupWithLegacyFrames(track, 1, [66_000 as Time.Micro, 50_000 as Time.Micro]); await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); for (const timestamp of [0, 33_000, 66_000, 50_000]) { - const result = await nextFrame(consumer); - expect(result?.frame?.timestamp).toBe(timestamp as Time.Micro); - expect(result?.discontinuity).toBe(0); + expect(frameOf(await consumer.next()).timestamp).toBe(timestamp as Time.Micro); } consumer.close(); @@ -1061,8 +1054,7 @@ test("Consumer preserves empty media from formats without endpoint markers", asy track.writeGroup(group); track.close(); - const result = await consumer.next(); - expect(result?.frame?.payload).toHaveLength(0); + expect(frameOf(await nextFrame(consumer)).payload).toHaveLength(0); consumer.close(); }); @@ -1256,9 +1248,7 @@ test("Consumer delivers a PTS-contiguous next group whose sequence jumped (CMAF) a.close(); track.writeGroup(a); - const firstFrame = await consumer.next(); - expect(firstFrame?.frame?.payload).toEqual(new Uint8Array([0x01])); - await consumer.next(); // group-done marker + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); const pending = consumer.next(); @@ -1285,7 +1275,7 @@ test("Consumer delivers a PTS-contiguous next group whose sequence jumped (CMAF) new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 500)), ]); expect(result).not.toBe("timeout"); - expect((result as { frame?: Frame } | undefined)?.frame?.payload).toEqual(new Uint8Array([0x02])); + expect(frameOf(result).payload).toEqual(new Uint8Array([0x02])); consumer.close(); }); @@ -1317,9 +1307,7 @@ test("Consumer waits on a PTS gap instead of skipping to a later buffered group a.close(); track.writeGroup(a); - const firstFrame = await consumer.next(); - expect(firstFrame?.frame?.payload).toEqual(new Uint8Array([0x01])); - await consumer.next(); // group-done marker + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); const pending = consumer.next(); @@ -1362,7 +1350,7 @@ test("Consumer waits on a PTS gap instead of skipping to a later buffered group }); const result = await pending; - expect((result as { frame?: Frame } | undefined)?.frame?.payload).toEqual(new Uint8Array([0x02])); + expect(frameOf(result).payload).toEqual(new Uint8Array([0x02])); consumer.close(); }); @@ -1383,7 +1371,7 @@ test("Consumer delivers a contiguous group after one that completed out of order track.writeGroup(a); a.writeFrame(cmafFrame(0x01, 0, 0)); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x01])); + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); // A's second frame stays buffered so A can't be duration-skipped. A ends at 6000 ticks (66_666µs). a.writeFrame(cmafFrame(0x02, 3000, 1)); @@ -1398,10 +1386,8 @@ test("Consumer delivers a contiguous group after one that completed out of order await settle(); a.close(); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x02])); - expect((await consumer.next())?.frame).toBeUndefined(); // A group-done - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x03])); - expect((await consumer.next())?.frame).toBeUndefined(); // B group-done + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x03])); // C (seq 1_090_000) starts at B's end, so it continues the timeline and must deliver. const pending = consumer.next(); @@ -1411,7 +1397,7 @@ test("Consumer delivers a contiguous group after one that completed out of order const result = await Promise.race([pending, settle(300).then(() => "timeout" as const)]); expect(result).not.toBe("timeout"); - expect((result as { frame?: Frame } | undefined)?.frame?.payload).toEqual(new Uint8Array([0x04])); + expect(frameOf(result).payload).toEqual(new Uint8Array([0x04])); consumer.close(); }); @@ -1430,9 +1416,8 @@ test("Consumer plays the head once a waited-out gap exceeds the budget (CMAF)", const a = new Group.Producer(1000); track.writeGroup(a); a.writeFrame(cmafFrame(0x01, 0, 0)); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x01])); + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); a.close(); - expect((await consumer.next())?.frame).toBeUndefined(); // #active falls back to the 1001 phantom // The reader is parked on the missing 1001, as a decoder is while it waits for the next group. const pending = consumer.next(); @@ -1443,12 +1428,10 @@ test("Consumer plays the head once a waited-out gap exceeds the budget (CMAF)", track.writeGroup(b); b.writeFrame(cmafFrame(0x02, 90_000, 1)); + // The missing group's span was never delivered, so a gap comes first. const result = await Promise.race([pending, settle(300).then(() => "timeout" as const)]); - expect(result).not.toBe("timeout"); - const delivered = result as { frame?: Frame; continuous?: boolean } | undefined; - expect(delivered?.frame?.payload).toEqual(new Uint8Array([0x02])); - // The missing group's span was never delivered, so downstream must not bridge it. - expect(delivered?.continuous).toBe(false); + expect(result).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); consumer.close(); }); @@ -1465,9 +1448,8 @@ test("Consumer waits on a gap within the budget, then plays the head once it exc const a = new Group.Producer(1000); track.writeGroup(a); a.writeFrame(cmafFrame(0x01, 0, 0)); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x01])); + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); a.close(); - expect((await consumer.next())?.frame).toBeUndefined(); // B (seq 2000) starts at 6000 ticks (66_667µs): a gap, but only 33ms past A's end. const b = new Group.Producer(2000); @@ -1485,36 +1467,36 @@ test("Consumer waits on a gap within the budget, then plays the head once it exc } const result = await Promise.race([pending, settle(300).then(() => "timeout" as const)]); - expect(result).not.toBe("timeout"); - expect((result as { frame?: Frame } | undefined)?.frame?.payload).toEqual(new Uint8Array([0x02])); + expect(result).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); consumer.close(); }); -// --- Continuity reporting --- +// --- Gaps and ends --- -// `continuous` is what downstream buffer accounting keys on, so it must be false exactly when the -// consumer dropped something. Group numbers can't answer that: they aren't required to be -// sequential, and adjacency doesn't rule out a group the max delay check dropped on the way past. -test("Consumer reports continuity while nothing is dropped", async () => { +// A gap is what downstream buffer accounting keys on, so it must appear exactly when the consumer +// dropped something. Group numbers can't answer that: they aren't required to be sequential, and +// adjacency doesn't rule out a group the max delay check dropped on the way past. +test("Consumer yields no gap while nothing is dropped", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 100 as Time.Milli }); writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 33_000 as Time.Micro]); await settle(); - expect((await consumer.next())?.continuous).toBe(false); // nothing precedes the first frame - expect((await consumer.next())?.continuous).toBe(true); // same group: consecutive by protocol - expect((await consumer.next())?.frame).toBeUndefined(); // group 0 done + expect(await consumer.next()).toEqual({ kind: "gap" }); // nothing precedes the first frame + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + expect(frameOf(await consumer.next()).timestamp).toBe(33_000 as Time.Micro); writeGroupWithLegacyFrames(track, 1, [66_000 as Time.Micro]); await settle(); - expect((await consumer.next())?.continuous).toBe(true); // group 1 continues group 0 + expect(frameOf(await consumer.next()).timestamp).toBe(66_000 as Time.Micro); // group 1 continues group 0 consumer.close(); }); -test("Consumer reports a marker group as a playhead event", async () => { +test("Consumer reads a marker group as an end, not a gap", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 2_000 as Time.Milli }); @@ -1523,13 +1505,11 @@ test("Consumer reports a marker group as a playhead event", async () => { writeGroupWithLegacyFrames(track, 2, [1_000_000 as Time.Micro]); await settle(); - expect((await consumer.next())?.discontinuity).toBe(0); - expect((await consumer.next())?.discontinuity).toBe(0); // group 0 done - const reset = await consumer.next(); - expect(reset?.frame).toBeUndefined(); - expect(reset?.discontinuity).toBe(1); - expect(reset?.continuous).toBe(false); - expect((await nextFrame(consumer))?.discontinuity).toBe(1); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + expect(await consumer.next()).toEqual({ kind: "end", group: 1, end: 0 as Time.Micro }); + // A declared stop loses nothing, so no gap follows it. + expect(frameOf(await consumer.next()).timestamp).toBe(1_000_000 as Time.Micro); consumer.close(); }); @@ -1545,52 +1525,48 @@ test("Consumer empty groups mean nothing", async () => { writeGroupWithLegacyFrames(track, 2, [1_000 as Time.Micro]); await settle(); - expect((await nextFrame(consumer))?.frame?.timestamp).toBe(0 as Time.Micro); - expect((await nextFrame(consumer))?.frame?.timestamp).toBe(1_000 as Time.Micro); - expect(consumer.discontinuity).toBe(0); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + expect(frameOf(await consumer.next()).timestamp).toBe(1_000 as Time.Micro); consumer.close(); }); -test("Consumer latency skip bumps playhead generation once", async () => { +test("Consumer reads a latency skip as one gap", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Time.Milli }); writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); await settle(); expect((await nextFrame(consumer))?.frame?.timestamp).toBe(0 as Time.Micro); - expect((await consumer.next())?.frame).toBeUndefined(); writeGroupWithLegacyFrames(track, 2, [100_000 as Time.Micro]); await settle(); - const resumed = await nextFrame(consumer); - expect(resumed?.frame?.timestamp).toBe(100_000 as Time.Micro); - expect(resumed?.discontinuity).toBe(1); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); consumer.close(); }); -test("Consumer jumps the playhead after a shed marker with a timestamp hole", async () => { +test("Consumer reads a timestamp hole as a gap", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Time.Milli }); writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); await settle(); - expect((await consumer.next())?.frame?.timestamp).toBe(0 as Time.Micro); - expect((await consumer.next())?.frame).toBeUndefined(); + expect((await nextFrame(consumer))?.frame?.timestamp).toBe(0 as Time.Micro); writeGroupWithLegacyFrames(track, 3, [1_000_000 as Time.Micro]); await settle(); - const resumed = await nextFrame(consumer); - expect(resumed?.frame?.timestamp).toBe(1_000_000 as Time.Micro); - expect(resumed?.discontinuity).toBe(1); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(1_000_000 as Time.Micro); consumer.close(); }); -test("Consumer zero-budget skip keeps a contiguous marker", async () => { +test("Consumer reads a shed marker group as a gap", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Time.Milli }); @@ -1607,16 +1583,16 @@ test("Consumer zero-budget skip keeps a contiguous marker", async () => { writeGroupWithLegacyFrames(track, 2, [500 as Time.Micro]); await settle(); - const resumed = await nextFrame(consumer); - expect(resumed?.frame?.timestamp).toBe(500 as Time.Micro); - expect(resumed?.discontinuity).toBe(1); + // The zero budget sheds the marker unread, and what it declared is lost with it. + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(500 as Time.Micro); group0.close(); consumer.close(); }); // The case the group-number heuristic got backwards: ids jump, but the PTS timeline is unbroken. -test("Consumer reports continuity across a PTS-contiguous group id jump (CMAF)", async () => { +test("Consumer yields no gap across a PTS-contiguous group id jump (CMAF)", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new CmafFormat(TEST_INIT), @@ -1628,23 +1604,19 @@ test("Consumer reports continuity across a PTS-contiguous group id jump (CMAF)", a.writeFrame(cmafFrame(0x01, 0, 0)); a.close(); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([0x01])); - expect((await consumer.next())?.frame).toBeUndefined(); // group-done + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([0x01])); // Sequence jumps +90_000 but the first PTS meets A's end, so nothing is missing. const b = new Group.Producer(1_090_000); track.writeGroup(b); b.writeFrame(cmafFrame(0x02, 3000, 1)); - const result = await consumer.next(); - expect(result?.frame?.payload).toEqual(new Uint8Array([0x02])); - expect(result?.continuous).toBe(true); - expect(result?.discontinuity).toBe(0); + expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); consumer.close(); }); -test("live duration marker follows an immediately delivered video frame", async () => { +test("a live video frame is delivered before its duration marker arrives", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("video"), @@ -1662,14 +1634,16 @@ test("live duration marker follows an immediately delivered video frame", async }); await settle(); expect(delivered).toBe(true); - expect((await read)?.frame?.timestamp).toBe(0 as Time.Micro); + expect(await read).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); group.writeFrame({ payload: encodeLegacyFrame(15_000 as Time.Micro, new Uint8Array()), timestamp: Time.Timestamp.now(), }); - const result = await consumer.next(); - expect(result?.frame).toBeUndefined(); - expect(result?.end).toBe(15_000 as Time.Micro); + group.close(); + track.close(); + // The marker is the frame's duration, not an item. + expect(await consumer.next()).toBeUndefined(); consumer.close(); track.close(); }); @@ -1700,18 +1674,18 @@ for (const code of [StreamCode.Cancel, StreamCode.Internal, StreamCode.Old, Stre await settle(); group.close(new NetError.Stream(code)); await settle(); - expect((await consumer.next())?.frame?.payload).toEqual(new Uint8Array([1])); - expect((await consumer.next())?.frame).toBeUndefined(); + expect(frameOf(await nextFrame(consumer)).payload).toEqual(new Uint8Array([1])); const next = track.appendGroup(); next.writeFrame({ payload: encodeLegacyFrame(1 as Time.Micro, new Uint8Array([2])), timestamp: Time.Timestamp.now(), }); next.close(); - const resumed = await consumer.next(); - expect(resumed?.frame?.payload).toEqual(new Uint8Array([2])); - expect(resumed?.continuous).toBe(false); - expect(resumed?.frame?.keyframe).toBe(true); + // The reset cut the group's tail. + expect(await consumer.next()).toEqual({ kind: "gap" }); + const resumed = frameOf(await consumer.next()); + expect(resumed.payload).toEqual(new Uint8Array([2])); + expect(resumed.keyframe).toBe(true); expect(error).not.toHaveBeenCalled(); } finally { consumer.close(); @@ -1775,8 +1749,7 @@ for (const end of [ }); try { writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); - expect((await consumer.next())?.group).toBe(0); - expect((await consumer.next())?.frame).toBeUndefined(); + expect(await nextFrame(consumer)).toMatchObject({ group: 0 }); writeGroupWithLegacyFrames(track, 2, [1_000_000 as Time.Micro]); await settle(); const pending = consumer.next(); @@ -1784,10 +1757,8 @@ for (const end of [ const resumed = await Promise.race([pending, settle(100).then(() => "stalled" as const)]); expect(resumed).not.toBe("stalled"); if (resumed === "stalled") return; - expect(resumed?.group).toBe(2); - expect(resumed?.frame?.timestamp).toBe(1_000_000 as Time.Micro); - expect(resumed?.continuous).toBe(false); - expect((await consumer.next())?.frame).toBeUndefined(); + expect(resumed).toEqual({ kind: "gap" }); + expect(await consumer.next()).toMatchObject({ kind: "frame", group: 2, frame: { timestamp: 1_000_000 } }); if (end) await expect(consumer.next()).rejects.toBe(end); else expect(await consumer.next()).toBeUndefined(); } finally { diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index a7e563d6fd2..a0670f027e3 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -21,6 +21,23 @@ export interface ConsumerProps { maxDelay?: GetterInit; } +/** One item of a {@link Consumer.next} read, in delivery order. Mirrors Rust's `container::Read`. */ +export type Read = + /** A media frame from `group`. Only group equality means anything: numbers may skip. */ + | { kind: "frame"; group: number; frame: Frame } + /** + * Media before the next item will never reach this reader: it joined mid-track, or a group was + * skipped, evicted, truncated, or never arrived. A decoder resumes without the references or + * state that media built. + */ + | { kind: "gap" } + /** + * The publisher declared that its media stops at `end`: it paused, switched source, or finished. + * Frames that follow from the same `group` are the stopped encoder's final packets (an audio + * codec's drain), whose output at or past `end` is padding. The next group starts fresh. + */ + | { kind: "end"; group: number; end: Time.Micro }; + interface Group { consumer: Moq.Group.Consumer; frames: Frame[]; // decode order @@ -78,21 +95,18 @@ export class Consumer { // promotion guard can tell a timeline-continuous next group from one sitting after a gap. // Maintained only via #recordPresented; see its comment for the invariant. #presentedEnd?: Time.Micro; - // Group of the last frame next() returned, so it can report whether the following result - // continues that frame's timeline. Undefined until the first delivery and after a playhead event. - #deliveredGroup?: number; - // Set whenever the consumer throws content away: a group that fell `maxDelay` behind, - // a group truncated by a decode error. Reported (and - // cleared) on the first frame delivered from the next group, which is where the missing span - // sits. Only the consumer can know this, which is why next() reports it instead of leaving - // callers to guess from group numbers. - #gap = false; + // Group of the last media frame next() returned, so an endpoint after it reads as that frame's + // duration rather than a declared stop. + #mediaGroup?: number; + // Media went missing since the last item read, so the next one is preceded by a gap. Starts set: + // nothing before the first frame was read. + #lost = true; + // The item a gap was returned ahead of. + #next?: Read; // The first media timestamp of the latest delivered group, and its predecessor's start. // Endpoints and later frames can overlap the next group without moving this floor. #start?: { group: number; timestamp: Time.Micro }; #floor?: Time.Micro; - // Increments on a declared marker, an unproven delivered hole, and a latency skip. - #discontinuity = 0; // A group below the previous group start aborts the track. #error?: Error; @@ -313,22 +327,11 @@ export class Consumer { if (group.end !== undefined) this.#presentedEnd = group.end; } - // Whether delivering from group `sequence` continues the timeline of the last frame returned. - // Frames within a group are consecutive by protocol, so only a group boundary can break it, and - // there it comes down to whether anything was dropped in between. Deliberately not derived from - // group numbers: they need not be sequential, so adjacency neither proves continuity nor catches - // a group the max delay check truncated on the way past. - #continuesDelivery(sequence: number): boolean { - if (this.#deliveredGroup === undefined) return false; - return sequence === this.#deliveredGroup || !this.#gap; - } - #checkMaxDelay() { if (this.#active === undefined) return; let skipped = 0; const start = this.#groups[0]?.consumer.sequence; - let hole = false; // Keep skipping the oldest group while the buffered span exceeds the max delay. // This also handles gaps in group sequence numbers: if #active points to a missing @@ -359,17 +362,16 @@ export class Consumer { const nextStart = this.#groups[0]?.frames.at(0)?.timestamp ?? this.#groups[0]?.end; const marker = !first.empty && !first.media; - if (marker || !ptsContiguous(first.end ?? this.#presentedEnd, nextStart)) { - hole = true; + // Buffered frames are thrown away, and `first.end` counts them, so contiguity alone would hide them. + const discarded = first.frames.length > 0; + if (marker || discarded || !ptsContiguous(first.end ?? this.#presentedEnd, nextStart)) { + this.#lost = true; } first.consumer.close(); first.frames.length = 0; skipped++; - this.#gap = true; } - if (hole) this.#markPlayhead(); - if (skipped) { console.warn( `skipping slow groups: track=${this.#track.name} ${start} -> ${this.#active} count=${skipped}`, @@ -441,35 +443,17 @@ export class Consumer { } /** - * Returns the next frame in order along with its group number and the current - * {@link discontinuity} count, awaiting one if needed. A `frame` of undefined signals either - * the end of that group or, when `end` is present, an exclusive media endpoint carried by a - * legacy marker. The overall result is undefined once closed. When `discontinuity` - * jumps relative to the previous call, re-apply startup delay and skip: it is a playhead - * event, and the next frame may be a delta that continues the interrupted group. - * - * `continuous` is true when this result picks up exactly where the previous frame left off, so - * the span between them can be treated as delivered. It is false on the first frame, after a - * playhead event, and whenever the consumer threw content away to keep up: a slow group skipped - * for the max delay, a group truncated by a decode error. Use it rather than comparing group - * numbers, which are not required to be sequential: adjacency neither proves the timeline is - * unbroken nor catches a group dropped on the way past. - * - * It reports what this consumer dropped plus marker groups the publisher declared. - * An unmarked forward timestamp jump still reads as continuous because nothing on the wire says - * the missing span will never arrive. + * Returns the next {@link Read} in delivery order, awaiting one if needed: a frame, a gap where + * media will never arrive (always one before the first frame), or an end the publisher declared. + * An unmarked forward timestamp jump is not a gap, since nothing on the wire says media is missing. * After buffered groups drain, a finished track returns undefined and an aborted track throws. */ - async next(): Promise< - | { - frame: Frame | undefined; - group: number; - discontinuity: number; - continuous: boolean; - end?: Time.Micro; - } - | undefined - > { + async next(): Promise { + if (this.#next) { + const next = this.#next; + this.#next = undefined; + return next; + } for (;;) { if (this.#error) throw this.#error; this.#checkMalformed(); @@ -487,8 +471,8 @@ export class Consumer { // fallback fired because no later group was buffered yet, and the real (large-gap, // non-sequential) next group has since arrived -- promote #active to the first real // group so delivery resumes instead of stalling on a nonexistent sequence. - // Before any media is delivered there is no timeline hole to wait for. #start survives - // playhead events, unlike #deliveredGroup, so this exception only applies at startup. + // Before any media is delivered there is no timeline hole to wait for, so this exception + // only applies at startup. // Otherwise promote #active when the first buffered group continues the timeline we left off at, // when a completed empty group can be walked (empty groups mean nothing), or when the hole // is proven: the head already reaches past where presentation left off by more than the @@ -511,9 +495,7 @@ export class Consumer { head.latest !== undefined && head.latest - this.#presentedEnd > maxDelay)); if (startup || empty || contiguous || skipHole || ended !== undefined) { - if (!startup && (skipHole || ended !== undefined) && !contiguous && !empty) - this.#markPlayhead(); - if (!contiguous) this.#gap = true; + if (!contiguous && !empty) this.#lost = true; this.#active = head.consumer.sequence; } } @@ -529,19 +511,12 @@ export class Consumer { const seq = this.#groups[0].consumer.sequence; const end = this.#format.end?.(frame); if (end !== undefined) { - if (!this.#groups[0].media) this.#markPlayhead(); this.#updateBuffered(); - return { - frame: undefined, - group: seq, - discontinuity: this.#discontinuity, - continuous: this.#continuesDelivery(seq), - end, - }; + // A trailing endpoint is the last frame's duration, which only the skip logic needs. + if (seq === this.#mediaGroup) continue; + return this.#emit({ kind: "end", group: seq, end }); } - const continuous = this.#continuesDelivery(seq); - if (seq !== this.#deliveredGroup) this.#gap = false; - this.#deliveredGroup = seq; + this.#mediaGroup = seq; if (this.#start === undefined || seq > this.#start.group) { this.#floor = this.#start?.timestamp; @@ -553,7 +528,7 @@ export class Consumer { throw this.#error; } this.#updateBuffered(); - return { frame, group: seq, discontinuity: this.#discontinuity, continuous }; + return this.#emit({ kind: "frame", group: seq, frame }); } // Check if the group is done and then remove it. A group is removable only @@ -575,17 +550,9 @@ export class Consumer { const group = this.#groups.shift(); if (group) { - const seq = group.consumer.sequence; - if (group.truncated) this.#gap = true; + if (group.truncated) this.#lost = true; this.#updateBuffered(); - return { - frame: undefined, - group: seq, - discontinuity: this.#discontinuity, - // A marker carries no content of its own, so this just reports whether - // the group it closes was itself reached without a gap. - continuous: this.#continuesDelivery(seq), - }; + continue; } } @@ -616,10 +583,12 @@ export class Consumer { } } - #markPlayhead(): void { - this.#discontinuity++; - this.#deliveredGroup = undefined; - this.#gap = true; + // Return `read`, or a gap ahead of it when media went missing first. + #emit(read: Read): Read { + if (!this.#lost) return read; + this.#lost = false; + this.#next = read; + return { kind: "gap" }; } #updateBuffered(): void { @@ -648,14 +617,6 @@ export class Consumer { this.#buffered.set(ranges); } - /** - * A counter that increments at each playhead event: a declared marker group, an unproven - * delivered hole, or a latency skip. Also surfaced per-read via {@link next}. - */ - get discontinuity(): number { - return this.#discontinuity; - } - /** Stop consuming and release the track and all buffered groups. */ close(): void { this.#finish(null); diff --git a/js/hang/src/container/index.ts b/js/hang/src/container/index.ts index 5ef19f7f7a8..135c5600d8f 100644 --- a/js/hang/src/container/index.ts +++ b/js/hang/src/container/index.ts @@ -7,7 +7,7 @@ export * as Loc from "@moq/loc"; export * as Cmaf from "./cmaf"; -export { Consumer, type ConsumerProps } from "./consumer"; +export { Consumer, type ConsumerProps, type Read } from "./consumer"; export type { Format } from "./format"; export * as Legacy from "./legacy"; export { type TrackInfoOptions, trackInfo } from "./track"; diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 77da93f5c88..15c8b468035 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -67,15 +67,14 @@ class FakeData { close() {} } -type Read = NonNullable>>; +type Read = Container.Read; let frameTimestamp = 0; -function frame(): Read { +function frame(): Extract { frameTimestamp += 20_000; return { + kind: "frame", group: 0, - discontinuity: 0, - continuous: true, frame: { timestamp: Time.Micro(frameTimestamp), payload: new Uint8Array([1]), keyframe: true }, }; } @@ -234,7 +233,7 @@ async function play(initial: Delay, buffered = false, cmaf = false) { const timestamps: number[] = []; for (let i = 0; i < count; i++) { const read = frame(); - timestamps.push(read.frame?.timestamp ?? 0); + timestamps.push(read.frame.timestamp); push(read); await microtasks(); } @@ -291,13 +290,12 @@ async function play(initial: Delay, buffered = false, cmaf = false) { }; } -it("decodes an audio packet inside a group after a playhead discontinuity", async () => { +it("decodes an audio packet inside a group after a gap", async () => { const playback = await play(Time.Milli(100)); try { await playback.play(); + playback.push({ kind: "gap" }); const next = frame(); - next.discontinuity = 1; - if (!next.frame) throw new Error("missing test frame"); next.frame.keyframe = false; playback.push(next); await microtasks(); diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 348f378ff96..d618f87db53 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -119,7 +119,7 @@ export class Decoder { // The rendition is absent and the ring has drained its tail, so the graph has nothing to play. readonly #idle: Computed; - // Ordered discontinuity and endpoint state from the container consumer. + // Ordered epoch and endpoint state from the container consumer. #terminal = new Terminal(); // The container consumer's arrival estimate, unset while nothing is subscribed. @@ -471,16 +471,12 @@ export class Decoder { for (;;) { const next = await nextMedia(consumer); if (!next) break; - if (this.#onNext(next)) { + if (this.#onRead(next)) { decoder.reset(); decoder.configure(decoderConfig); } - if (next.end !== undefined) { - continue; - } - + if (next.kind !== "frame") continue; const { frame } = next; - if (!frame) continue; // Mark that we received this frame right now. const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro); @@ -581,13 +577,12 @@ export class Decoder { if (!next) break; // Reset and re-anchor before decoding the first frame of a new codec epoch. - if (this.#onNext(next)) { + if (this.#onRead(next)) { decoder.reset(); decoder.configure(decoderConfig); } - + if (next.kind !== "frame") continue; const { frame } = next; - if (!frame) continue; const timestamp = Time.Milli.fromMicro(frame.timestamp); this.sync.received(timestamp, "audio"); @@ -732,15 +727,10 @@ export class Decoder { this.#ring.peek()?.reset(); } - // Apply ordered container metadata before handling the result. An endpoint that also - // starts a new epoch must survive the reset so its following drain is trimmed. - #onNext(next: { - discontinuity: number; - group: number; - end?: Time.Micro; - frame?: { timestamp: Time.Micro }; - }): boolean { - if (!this.#terminal.update(next)) return false; + // Apply an ordered container read before handling it, reporting whether a codec epoch ended: + // a gap, or a later group after a declared end. + #onRead(read: Container.Read): boolean { + if (!this.#terminal.update(read)) return false; this.#ring.peek()?.reset(); this.sync.reset(); return true; diff --git a/js/watch/src/audio/playout.test.ts b/js/watch/src/audio/playout.test.ts index 362681f2186..e2e05d9e944 100644 --- a/js/watch/src/audio/playout.test.ts +++ b/js/watch/src/audio/playout.test.ts @@ -89,7 +89,7 @@ async function measure(t: Arrival[], budget: (target: Time.Milli) => Time.Milli) for (;;) { const next = await consumer.next(); if (!next) return; - if (next.frame) result.delivered++; + if (next.kind === "frame") result.delivered++; } })(); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index 3c18faf97ba..e7ae9424f08 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -3,7 +3,7 @@ * * Every frame is written into its group at the instant the trace says it arrived, and the player's * real `Container.Consumer` decides what to deliver, wait for, and skip, at the max delay the delay - * sets. Delivered frames go into the real ring, and a discontinuity resets it, the way the decoder + * sets. Delivered frames go into the real ring, and a new codec epoch resets it, the way the decoder * does. One render quantum is read every quantum's worth of that same clock, which is what the * AudioWorklet does. A real {@link Sync} resolves the delay from the playout target the decoder * registers, measured by that same consumer from the recorded catalog config, and the ring follows @@ -221,7 +221,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const next = await consumer.next(); if (!next) return; if (terminal.update(next)) ring.reset(); - if (next.end !== undefined || !next.frame) continue; + if (next.kind !== "frame") continue; const { timestamp } = next.frame; const length = samples.get(timestamp); diff --git a/js/watch/src/audio/terminal.test.ts b/js/watch/src/audio/terminal.test.ts index 6c6d9607cad..44eb0d06c33 100644 --- a/js/watch/src/audio/terminal.test.ts +++ b/js/watch/src/audio/terminal.test.ts @@ -2,21 +2,31 @@ import { expect, test } from "bun:test"; import type { Time } from "@moq/net"; import { Terminal } from "./terminal"; +const frame = (group: number, timestamp: number) => ({ + kind: "frame" as const, + group, + frame: { timestamp: timestamp as Time.Micro }, +}); +const end = (group: number, timestamp: number) => ({ kind: "end" as const, group, end: timestamp as Time.Micro }); +const gap = { kind: "gap" as const }; + test("Terminal trims a partial packet to the source endpoint", () => { const terminal = new Terminal(); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro }, end: 17_916 as Time.Micro }); + terminal.update(end(0, 17_916)); + terminal.update(frame(0, 0)); expect(terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(860); }); test("Terminal maps Opus delay back before trimming terminal output", () => { const terminal = new Terminal(); terminal.clear(312); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro } }); + terminal.update(frame(0, 0)); const body = terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }); expect(body).toEqual({ timestamp: 0 as Time.Micro, frameOffset: 312, frames: 648 }); - terminal.update({ discontinuity: 0, group: 0, end: 20_000 as Time.Micro }); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 20_000 as Time.Micro } }); + expect(terminal.update(end(1, 20_000))).toBe(false); + // The drain shares the end's group, so it continues the epoch. + expect(terminal.update(frame(1, 20_000))).toBe(false); const drain = terminal.span({ timestamp: 20_000, sampleRate: 48_000, numberOfFrames: 960 }); expect(drain).toEqual({ timestamp: 13_500 as Time.Micro, frameOffset: 0, frames: 312 }); expect(body.frames + drain.frames).toBe(960); @@ -24,41 +34,48 @@ test("Terminal maps Opus delay back before trimming terminal output", () => { test("Terminal preserves samples before the source endpoint", () => { const terminal = new Terminal(); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro }, end: 20_000 as Time.Micro }); + terminal.update(end(0, 20_000)); + terminal.update(frame(0, 0)); expect(terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(960); }); -test("a rewound endpoint survives the discontinuity reset and trims its terminal packet", () => { +test("the gap before the first frame ends no epoch", () => { + const terminal = new Terminal(); + expect(terminal.update(gap)).toBe(false); +}); + +test("an endpoint after a gap trims its own group", () => { const terminal = new Terminal(); terminal.clear(312); - terminal.update({ discontinuity: 0, group: 0, end: 40_000 as Time.Micro }); + terminal.update(frame(0, 0)); + expect(terminal.update(gap)).toBe(true); - const reset = terminal.update({ discontinuity: 1, group: 1, end: 20_000 as Time.Micro }); - expect(reset).toBe(true); + terminal.update(end(1, 20_000)); expect(terminal.end).toBe(20_000 as Time.Micro); - - terminal.update({ discontinuity: 1, group: 1, frame: { timestamp: 20_000 as Time.Micro } }); + terminal.update(frame(1, 20_000)); const span = terminal.span({ timestamp: 20_000, sampleRate: 48_000, numberOfFrames: 960 }); expect(span).toEqual({ timestamp: 20_000 as Time.Micro, frameOffset: 312, frames: 0 }); }); -test("a discontinuity reapplies Opus pre-skip in the resumed epoch", () => { +test("a gap reapplies Opus pre-skip in the resumed epoch", () => { const terminal = new Terminal(); terminal.clear(312); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro } }); + terminal.update(frame(0, 0)); expect(terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }).frameOffset).toBe(312); - terminal.update({ discontinuity: 1, group: 1, frame: { timestamp: 1_000_000 as Time.Micro } }); + expect(terminal.update(gap)).toBe(true); + terminal.update(frame(1, 1_000_000)); const resumed = terminal.span({ timestamp: 1_000_000, sampleRate: 48_000, numberOfFrames: 960 }); expect(resumed).toEqual({ timestamp: 1_000_000 as Time.Micro, frameOffset: 312, frames: 648 }); }); test("an endpoint does not trim media resumed in a later group", () => { const terminal = new Terminal(); - terminal.update({ discontinuity: 0, group: 0, frame: { timestamp: 0 as Time.Micro } }); + terminal.update(frame(0, 0)); // A discontinuity marker: its own group, ending the epoch at the live edge. - terminal.update({ discontinuity: 1, group: 1, end: 20_000 as Time.Micro }); - terminal.update({ discontinuity: 1, group: 2, frame: { timestamp: 5_000_000 as Time.Micro } }); + expect(terminal.update(end(1, 20_000))).toBe(false); + // The next group comes from a fresh encoder. + expect(terminal.update(frame(2, 5_000_000))).toBe(true); expect(terminal.end).toBeUndefined(); expect(terminal.span({ timestamp: 5_000_000, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(960); }); diff --git a/js/watch/src/audio/terminal.ts b/js/watch/src/audio/terminal.ts index f89a8b15e74..59e501ae176 100644 --- a/js/watch/src/audio/terminal.ts +++ b/js/watch/src/audio/terminal.ts @@ -6,12 +6,11 @@ interface SampleSpan { readonly numberOfFrames: number; } -interface Update { - readonly discontinuity: number; - readonly group: number; - readonly end?: Time.Micro; - readonly frame?: { readonly timestamp: Time.Micro }; -} +/** The parts of a container read the terminal tracks. */ +type Update = + | { readonly kind: "frame"; readonly group: number; readonly frame: { readonly timestamp: Time.Micro } } + | { readonly kind: "gap" } + | { readonly kind: "end"; readonly group: number; readonly end: Time.Micro }; /** The portion of decoded audio that belongs to the source timeline. */ export interface DecodedSpan { @@ -25,7 +24,8 @@ export interface DecodedSpan { /** Track codec epochs and map decoded output onto the source timeline. */ export class Terminal { - #discontinuity = 0; + // Whether a frame arrived since the last reset, so there is an epoch to end. + #started = false; #end?: Time.Micro; // The group that carried `#end`. It bounds only the terminal packets that follow it there. #endGroup?: number; @@ -40,30 +40,39 @@ export class Terminal { /** Reset state for a new container consumer with Opus pre-skip in 48 kHz frames. */ clear(preSkip = 0): void { - this.#discontinuity = 0; + this.#started = false; this.#end = undefined; + this.#endGroup = undefined; this.#preSkip = preSkip; this.#resetEpoch(); } - /** Apply one ordered consumer result and report whether its codec epoch changed. */ - update(next: Update): boolean { - const reset = next.discontinuity !== this.#discontinuity; - if (reset) { - this.#discontinuity = next.discontinuity; - this.#end = undefined; - this.#resetEpoch(); - } - // A frame from a later group is new media, e.g. resuming after a discontinuity marker. - if (next.frame && next.group !== this.#endGroup) this.#end = undefined; - if (next.frame && this.#epoch === undefined) this.#epoch = next.frame.timestamp; - if (next.end !== undefined) { - this.#end = next.end; - this.#endGroup = next.group; + /** Apply one ordered consumer read and report whether a codec epoch ended before it. */ + update(read: Update): boolean { + if (read.kind === "gap") return this.#restart(); + if (read.kind === "end") { + this.#end = read.end; + this.#endGroup = read.group; + return false; } + // A later group after a declared end comes from a fresh encoder; the end's own group carries + // the stopped encoder's final packets. + const reset = this.#end !== undefined && read.group !== this.#endGroup && this.#restart(); + this.#started = true; + this.#epoch ??= read.frame.timestamp; return reset; } + // Start a new epoch, reporting whether one was running. + #restart(): boolean { + const started = this.#started; + this.#started = false; + this.#end = undefined; + this.#endGroup = undefined; + this.#resetEpoch(); + return started; + } + /** Remove codec pre-skip and terminal padding from one decoded sample. */ span(sample: SampleSpan): DecodedSpan { const epoch = this.#epoch ?? (sample.timestamp as Time.Micro); diff --git a/js/watch/src/retention.test.ts b/js/watch/src/retention.test.ts index 453c545f137..457b2ec975d 100644 --- a/js/watch/src/retention.test.ts +++ b/js/watch/src/retention.test.ts @@ -100,10 +100,10 @@ test("a long subscription through the player path leaves nothing behind", async const play = async (count: number) => { for (let i = 0; i < count; i++) { producer.encode(new Uint8Array([i & 0xff]), Math.round(Time.Micro.now()) as Time.Micro, true); - // A new group closes the previous one with a duration marker, which reads as no frame. + // Pass over the gap before the first frame. let next = await nextMedia(consumer); - while (next && !next.frame) next = await nextMedia(consumer); - if (!next?.frame) throw new Error("the track ended"); + while (next && next.kind !== "frame") next = await nextMedia(consumer); + if (!next) throw new Error("the track ended"); const timestamp = Time.Milli.fromMicro(next.frame.timestamp); sync.received(timestamp); diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index d18bc4775be..e2acb831415 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -173,15 +173,15 @@ class Picture { } } -type Read = NonNullable>>; -function sample(group: number, timestamp: number, discontinuity = 0, keyframe = true): Read { +type Read = Container.Read; +function sample(group: number, timestamp: number, keyframe = true): Read { return { + kind: "frame", group, - discontinuity, - continuous: true, frame: { timestamp: Time.Micro(timestamp), payload: new Uint8Array([1]), keyframe }, }; } +const gap: Read = { kind: "gap" }; async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: readonly number[] = []) { const originalDecoder = Object.getOwnPropertyDescriptor(globalThis, "VideoDecoder"); @@ -420,20 +420,20 @@ it("clears the picture once no rendition is selectable", async () => { } }); -// A late marker from older history raises a discontinuity without making older groups newer. -function marker(group: number, end: number, discontinuity: number): Read { - return { group, discontinuity, continuous: false, frame: undefined, end: Time.Micro(end) }; +// A late marker from older history breaks the timeline without making older groups newer. +function marker(group: number, end: number): Read { + return { kind: "end", group, end: Time.Micro(end) }; } for (const kind of ["legacy", "cmaf"] as const) { - it(`${kind} skips older groups before decode, even across a stale marker's discontinuity`, async () => { + it(`${kind} skips older groups before decode, even across a stale marker`, async () => { const playback = await guardedPlayback(kind, [ sample(10, 1000), sample(9, 900), sample(10, 1100), - marker(8, 850, 1), - sample(7, 700, 1), - sample(11, 1200, 1), + marker(8, 850), + sample(7, 700), + sample(11, 1200), ]); try { expect(playback.submitted).toEqual([1000, 1100, 1200]); @@ -446,9 +446,9 @@ for (const kind of ["legacy", "cmaf"] as const) { // A group may restart at its own start, so a picture already queued from later in that // group is above the new keyframe. It must not come out afterwards and late-reject it. - it(`${kind} drops a picture queued before a discontinuity`, async () => { + it(`${kind} drops a picture queued before a gap`, async () => { const queued = 60_000_000; - const playback = await guardedPlayback(kind, [sample(1, 0), sample(1, queued), sample(2, 0, 1)], [queued]); + const playback = await guardedPlayback(kind, [sample(1, 0), sample(1, queued), gap, sample(2, 0)], [queued]); try { expect(playback.submitted).toEqual([0, queued, 0]); expect(playback.outputs).toEqual([0, 0]); @@ -459,15 +459,15 @@ for (const kind of ["legacy", "cmaf"] as const) { } }); - // A stale marker (or a max-delay hole) can bump the playhead while the live group is only - // partly delivered, so the next frame is a delta that still needs the decoder's references. - it(`${kind} keeps decoding a delta that follows a discontinuity`, async () => { + // A stale marker can break the timeline while the live group is only partly delivered, so the + // next frame is a delta that still needs the decoder's references. + it(`${kind} keeps decoding a delta that follows a stale marker`, async () => { const playback = await guardedPlayback(kind, [ sample(10, 1000), - sample(10, 1100, 0, false), - marker(8, 850, 1), - sample(10, 1200, 1, false), - sample(11, 1300, 1), + sample(10, 1100, false), + marker(8, 850), + sample(10, 1200, false), + sample(11, 1300), ]); try { expect(playback.submitted).toEqual([1000, 1100, 1200, 1300]); @@ -603,7 +603,7 @@ it("a shared audio clock reset cannot present a parked video frame", async () => playback.enabled.set(false); playback.delay.set(Time.Milli(1000)); await microtasks(); - reads.push(sample(0, 0), sample(0, 100_000, 0, false)); + reads.push(sample(0, 0), sample(0, 100_000, false)); playback.enabled.set(true); await microtasks(); expect(playback.submitted).toEqual([0, 100_000]); @@ -645,7 +645,7 @@ it("closing the track releases a video wait parked across a reset", async () => playback.enabled.set(false); playback.delay.set(Time.Milli(1000)); await microtasks(); - reads.push(sample(0, 0), sample(0, 100_000, 0, false)); + reads.push(sample(0, 0), sample(0, 100_000, false)); playback.enabled.set(true); await microtasks(); expect(playback.submitted).toEqual([0, 100_000]); @@ -680,7 +680,7 @@ it("a reset drops a parked video frame even when the next anchor restores the re playback.enabled.set(false); playback.delay.set(Time.Milli(1000)); await microtasks(); - reads.push(sample(0, 0), sample(0, 100_000, 0, false)); + reads.push(sample(0, 0), sample(0, 100_000, false)); playback.enabled.set(true); await microtasks(); expect(playback.submitted).toEqual([0, 100_000]); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index a1f915fee1f..173047d44f0 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -410,9 +410,6 @@ class DecoderTrack { // Decoded frames waiting to be rendered. #buffered = new Signal([]); - // The last discontinuity count seen from the container consumer. - #discontinuity = 0; - // Codec description from the last configure, reused after a playhead reset. #description?: Uint8Array; @@ -543,15 +540,18 @@ class DecoderTrack { const next = await nextMedia(consumer); if (!next) break; - // Playhead moved: re-anchor before decoding further. - if (this.#onDiscontinuity(next.discontinuity)) previous = undefined; - + // A gap or a declared end breaks the timeline: re-anchor before decoding further, and + // never bridge the buffer across it. + if (next.kind !== "frame") { + if (latest !== undefined) this.#onBreak(); + previous = undefined; + continue; + } const { frame } = next; - if (!frame) continue; // The group is done // An older keyframe would replace the codec references needed by live deltas. Group - // sequences only grow within a subscription, so a discontinuity (which a stale marker - // can raise) never resets this guard. Rejected payloads still count as received bytes. + // sequences only grow within a subscription, so a break (which a stale marker can + // raise) never resets this guard. Rejected payloads still count as received bytes. const stale = latest !== undefined && next.group < latest; this.stats.update((current) => ({ frameCount: (current?.frameCount ?? 0) + (stale ? 0 : 1), @@ -570,11 +570,11 @@ class DecoderTrack { }); // Track decode buffer: frames sent to decoder but not yet rendered. Only bridge from - // the previous frame when the consumer says nothing is missing in between. Group ids - // can't answer that: they aren't required to be sequential (some encoders derive them - // from DTS), so adjacency neither proves continuity nor rules out a gap the consumer - // skipped, and reporting a skipped span as decoded overstates the buffer. - if (previous !== undefined && next.continuous) { + // the previous frame when no gap or end came between them. Group ids can't answer that: + // they aren't required to be sequential (some encoders derive them from DTS), so + // adjacency neither proves continuity nor rules out a gap the consumer skipped, and + // reporting a skipped span as decoded overstates the buffer. + if (previous !== undefined) { this.#addBuffered(Time.Milli.fromMicro(previous), Time.Milli.fromMicro(frame.timestamp)); } @@ -623,15 +623,17 @@ class DecoderTrack { const next = await nextMedia(consumer); if (!next) break; - // Playhead moved: re-anchor before decoding further. - if (this.#onDiscontinuity(next.discontinuity)) previous = undefined; - + // A gap or a declared end breaks the timeline (see #runLegacy). + if (next.kind !== "frame") { + if (latest !== undefined) this.#onBreak(); + previous = undefined; + continue; + } const { frame } = next; - if (!frame) continue; // An older keyframe would replace the codec references needed by live deltas. Group - // sequences only grow within a subscription, so a discontinuity (which a stale marker - // can raise) never resets this guard. Rejected payloads still count as received bytes. + // sequences only grow within a subscription, so a break (which a stale marker can + // raise) never resets this guard. Rejected payloads still count as received bytes. const stale = latest !== undefined && next.group < latest; this.stats.update((current) => ({ frameCount: (current?.frameCount ?? 0) + (stale ? 0 : 1), @@ -643,9 +645,9 @@ class DecoderTrack { const timestamp = Time.Milli.fromMicro(frame.timestamp); this.sync.received(timestamp, "video"); - // Track decode buffer (see #runLegacy: bridge on the consumer's continuity signal, + // Track decode buffer (see #runLegacy: bridge only when no gap or end came between, // never on group adjacency, which proves nothing about the timeline). - if (previous !== undefined && next.continuous) { + if (previous !== undefined) { this.#addBuffered(Time.Milli.fromMicro(previous), Time.Milli.fromMicro(frame.timestamp)); } @@ -665,25 +667,21 @@ class DecoderTrack { }); } - // The container bumped its playhead (a marker, a hole, or a latency skip). Re-anchor and - // let the next decoded frame decide the decoder reset (see #resetAtKeyframe). Clearing - // `timestamp` keeps a stale high value from late-rejecting the following frames, which may - // overlap the previous group's tail. The held picture stays until the new keyframe renders. - // Returns true when a new generation was handled. - #onDiscontinuity(count: number): boolean { - if (count === this.#discontinuity) return false; - this.#discontinuity = count; + // The container read a gap or a declared end after media. Re-anchor and let the next decoded + // frame decide the decoder reset (see #resetAtKeyframe). Clearing `timestamp` keeps a stale + // high value from late-rejecting the following frames, which may overlap the previous group's + // tail. The held picture stays until the new keyframe renders. + #onBreak(): void { this.timestamp.set(undefined); this.#buffered.set([]); this.sync.reset(); this.#resetPending = true; - return true; } - // Call before each decode. After a playhead bump, a keyframe starts a new timeline, so drop - // the pictures still queued in the decoder: one from later in a group that restarts would + // Call before each decode. After a break, a keyframe starts a new timeline, so drop the + // pictures still queued in the decoder: one from later in a group that restarts would // otherwise come out above the new keyframe and late-reject it. A delta continues a group the - // bump interrupted (a stale marker or a max-delay hole), so it keeps the decoder's references; + // break interrupted (a stale marker or a max-delay hole), so it keeps the decoder's references; // a reset would make WebCodecs require a key chunk and throw. `reset()` cannot cancel an // output callback that already holds a frame; the clock epoch it captured rejects that one. #resetAtKeyframe(decoder: VideoDecoder, keyframe: boolean): void { diff --git a/quest/m1/README.md b/quest/m1/README.md index ad8ca3fbe73..b65c148aeef 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -167,7 +167,6 @@ folded into Dropped sources. - [Encoder colour](/quest/m1/color-model.md) - every moq-video encode path signals the colour its output actually has, or refuses instead of mislabelling - [T-STD TS export](/quest/m1/tstd/README.md) - `moq export ts` is a proper remux that passes the T-STD buffer model, starting with a fixed `--delay` - [ATSC AC-3](/quest/m1/ts-atsc-ac3.md) - `export ts` carries 48 kHz ATSC AC-3 at every A/52 rate up to 640 kb/s instead of aborting from 384 kb/s up -- [Rust video delivery continuity](/quest/m1/video-continuous.md) - Rust video decode reads report each picture's delivery continuity, like the container and audio reads, for the tune-in and warmup trims - [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - a viewer joining at a recovery point drops the leading pictures it cannot decode; continuous viewers keep them - [Audio warmup](/quest/m1/audio-warmup.md) - a viewer joining an Opus rendition mid-stream never hears the unconverged first 80 ms - [#3021](/quest/m1/3021-moq-gst-anchor-generated-media-timelines-to-wall-clock.md) - moq-gst picks the broadcast wall epoch; a restarted source is a new epoch, not a forward re-anchor diff --git a/quest/m1/audio-warmup.md b/quest/m1/audio-warmup.md index 02c92f3a1ac..1f8888b360a 100644 --- a/quest/m1/audio-warmup.md +++ b/quest/m1/audio-warmup.md @@ -14,9 +14,9 @@ frames decode independently and set nothing; HE-AAC is out of scope. - `rs/moq-audio/src/encode/producer.rs` and `rs/moq-mux/src/codec/opus` publish `warmup` for Opus; `js/publish` does the same for its Opus track. - `rs/moq-audio/src/decode` already trims Opus `pre_skip` at stream start; - the warmup trim is the same mechanism keyed on the container consumer's - non-continuous signal (`continuous` on the Rust audio decoder's - `container::Delivery`), and the + the warmup trim is the same mechanism keyed on the container read's gap + item (`container::Read::Gap` in Rust, `{ kind: "gap" }` in JS), never on a + declared end, which would cut 80 ms at every pause, and the subscription's maximum age grows by `warmup` as the video consumer quest does. `js/watch` audio mirrors it. - The trim replaces `LEGACY_WARMUP_CALLBACKS` in `js/watch/src/audio/decoder.ts`, diff --git a/quest/m1/open-gop-leading-pictures.md b/quest/m1/open-gop-leading-pictures.md index 4ad437f5261..1fa124841f3 100644 --- a/quest/m1/open-gop-leading-pictures.md +++ b/quest/m1/open-gop-leading-pictures.md @@ -39,19 +39,16 @@ decode, and the JS test should drive the software decoder. - In the JS consumer (`js/hang/src/container/consumer.ts` forces the first sample of a group to `keyframe`, and `js/watch/src/video/decoder.ts` submits - it as `"key"`): for the first group after any non-continuous transition, - skip delta frames stamped before that group's keyframe. That covers a - subscribe, a declared discontinuity, and a latency skip: `#checkMaxDelay` - records the skip through `#gap` and `next()` reports the next frame with - `continuous: false`. Latency skip also bumps playhead generation (startup - delay) but does not flush the decoder. Leading pictures after that - non-continuous transition are still skipped, as above; a viewer that skipped - into a later GOP lacks its references just like a cold join. - Every continuous group is passed through untouched. + it as `"key"`): for the first group after a gap item, skip delta frames + stamped before that group's keyframe. `next()` yields `{ kind: "gap" }` + before the first frame of a subscribe and after a latency skip, eviction, or + hole; a declared end is not a gap, since a fresh encoder has no earlier GOP + to reference. A latency skip does not flush the decoder, and a viewer that + skipped into a later GOP lacks its references just like a cold join. + Every group without a gap before it is passed through untouched. - The same rule in the Rust decode path (`moq-video` decode consumers), so native playback and the transcoder tune in the same way, keyed on the - non-continuous signal from [Rust video delivery - continuity](/quest/m1/video-continuous.md). + container read's `Read::Gap` item. - The same rule in `moq export ts`. Decided (2026-10-01): the fixed-delay export still sends a join's orphaned leading pictures, so 3 of 500 frames on the open-GOP fixture decode after they present @@ -70,12 +67,8 @@ decode, and the JS test should drive the software decoder. group's leading pictures trimmed too, so an implementation that only trims the initial group fails. Both cases in both languages. -Decided 2026-10-08: the Rust non-continuous signal is its own quest, so -audio warmup and consumer warmup don't wait on this one. - -## Required - -- [Rust video delivery continuity](/quest/m1/video-continuous.md) - the Rust decode path keys the trim on it +Decided 2026-10-12: both container reads yield the gap item, so nothing +here waits on another quest. ## Related diff --git a/quest/m1/video-continuous.md b/quest/m1/video-continuous.md deleted file mode 100644 index 2c3b47db74d..00000000000 --- a/quest/m1/video-continuous.md +++ /dev/null @@ -1,39 +0,0 @@ -# [M] Rust video delivery continuity - -## Goal - -`moq_video::decode::Consumer::read` returns `container::Delivery`, so -each decoded picture says whether it continues the previous delivery, like -the container and audio reads already do. A picture decoded from input before -a marker, skip, or subscribe keeps that input's flag even when the codec -returns it late, and the first picture after the boundary reads -non-continuous. - -## Plan - -Split from the container and audio work in -[#5288](https://github.com/moq-dev/moq/pull/5288), where the maintainer chose -the `container::Delivery { frame, continuous }` wrapper and landing video -separately. - -A decoder can return a picture from before a marker while processing the -access unit after it, or only on flush, so a single pending boolean attaches -the boundary to the wrong picture. Carry a private input identity through -every backend (`rs/moq-video/src/decode/backend`) and map each output back to -its input's flag. A map keyed only by presentation time is not enough: equal -timestamps can straddle a marker. VideoToolbox and Media Foundation already -map synthetic sample times, OpenH264 queues presentation times, and VAAPI, -NVDEC and MediaCodec carry codec timestamps. An output whose input identity -did not survive (Media Foundation's unmatched-output fallback) is an explicit -error, never a borrowed flag. Keep dropped-input bookkeeping bounded. - -Tests: delayed output on both sides of a marker, duplicate timestamps, and -decoder recovery, on each backend CI can run. - -Public API: breaking in moq-video (the read result). Bindings keep their -existing frame records. Wire: none. - -## Related - -- [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - the Rust decode trim keys on this signal -- [Consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md) - withholds frames after the same signal diff --git a/quest/m2/intra-refresh/consumer-warmup.md b/quest/m2/intra-refresh/consumer-warmup.md index 350a6951ae3..e7c59349ef2 100644 --- a/quest/m2/intra-refresh/consumer-warmup.md +++ b/quest/m2/intra-refresh/consumer-warmup.md @@ -19,14 +19,12 @@ This absorbs the old gradual-recovery quest, which planned to carry `recovery_frame_cnt` on the wire and count `frame_num`. The catalog duration replaces both: the rule is timestamp arithmetic on the group start. -- Withhold rule, keyed on the same non-continuous signal in both consumers: - `js/hang/src/container/consumer.ts` `next()` reports `continuous: false` - after a subscribe, a declared discontinuity, or any skip (`#gap`). The Rust - `moq_mux::container::Consumer` reports the equivalent as - `container::Delivery::continuous`; this quest reuses it once - `rs/moq-video/src/decode/consumer.rs` propagates it in - [Rust video delivery continuity](/quest/m1/video-continuous.md). For the first - group after that signal, every frame is decoded (the decoder needs them to +- Withhold rule, keyed on the gap item both container reads yield before the + first frame and after any skip: `{ kind: "gap" }` from + `js/hang/src/container/consumer.ts` `next()` and `Read::Gap` from the Rust + `moq_mux::container::Consumer`, which `rs/moq-video/src/decode/consumer.rs` + reads directly. A declared end is not a gap and keeps everything. For the + first group after a gap, every frame is decoded (the decoder needs them to build reference state) and frames stamped below `group.start + warmup` are not presented; frames stamped at or above that boundary are, so the recovery picture itself is never withheld. `js/watch/src/video/decoder.ts` and @@ -57,10 +55,6 @@ replaces both: the rule is timestamp arithmetic on the group start. a group opening on an IDR presents immediately; a rendition without `warmup` is untouched. -## Required - -- [Rust video delivery continuity](/quest/m1/video-continuous.md) - the Rust video signal this keys on - ## Related - [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - trims frames stamped before the keyframe where this trims frames after the start diff --git a/rs/hang/examples/subscribe.rs b/rs/hang/examples/subscribe.rs index 9622449b2e0..f8fdb09e2f7 100644 --- a/rs/hang/examples/subscribe.rs +++ b/rs/hang/examples/subscribe.rs @@ -115,13 +115,17 @@ async fn play(consumer: moq_net::origin::Consumer, announce: moq_net::announce:: moq_mux::container::Consumer::new(track_consumer, moq_mux::catalog::hang::Container::try_from(config)?); // Read frames in latency-bounded presentation order. - while let Some(frame) = ordered.read().await? { - tracing::info!( - timestamp = ?frame.frame.timestamp, - keyframe = frame.frame.keyframe, - bytes = frame.frame.payload.len(), - "received frame" - ); + while let Some(read) = ordered.read().await? { + match read { + moq_mux::container::Read::Frame { frame, .. } => tracing::info!( + timestamp = ?frame.timestamp, + keyframe = frame.keyframe, + bytes = frame.payload.len(), + "received frame" + ), + moq_mux::container::Read::Gap => tracing::info!("media skipped"), + moq_mux::container::Read::End { end, .. } => tracing::info!(?end, "media stopped"), + } } Ok(()) diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 50abb09d301..80e7585b265 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -1,7 +1,8 @@ //! Subscribe to an encoded audio track and emit raw PCM. -use moq_mux::container::Delivery; use std::collections::VecDeque; + +use moq_mux::container::Read; use std::time::{Duration, Instant}; use bytes::Bytes; @@ -67,8 +68,8 @@ impl Options { /// /// The mirror of [`encode::Producer`](crate::encode::Producer): output format / /// sample rate / layout are fixed at construction, and -/// [`read`](Self::read) returns deliveries carrying each frame's continuity -/// and codec activity. +/// [`read`](Self::read) returns [`Frame`]s carrying the codec activity they +/// were decoded from. pub struct Consumer { decoder: Decoder, track: moq_mux::container::Consumer, @@ -85,7 +86,7 @@ pub struct Consumer { next_start: Option, /// Frames decoded and not yet handed back, so a gap's tail can be returned /// ahead of the packet that exposed it. - ready: VecDeque>, + ready: VecDeque, /// Codec activity spans the resampler's buffered output still covers. spans: VecDeque, /// Activity of the last span the output ran past, for the rounding samples the @@ -100,11 +101,10 @@ pub struct Consumer { frames_decoded: usize, /// Logical endpoint carried by an empty legacy frame before terminal packets. end: Option, + /// The group `end` came from: it bounds only the terminal packets that follow it there. + end_group: u64, /// Presentation time of the first decoded terminal frame. terminal_start: Option, - /// Last container playhead generation applied to timeline state. - discontinuity: u64, - continuous: bool, /// The playout delay, fixed or estimated from arrivals. target: Target, /// Where arrival times are measured from. @@ -208,9 +208,8 @@ impl Consumer { delay_trimmed: 0, frames_decoded: 0, end: None, + end_group: 0, terminal_start: None, - discontinuity: 0, - continuous: false, origin: now(), }) } @@ -253,7 +252,6 @@ impl Consumer { } /// Read the next decoded PCM frame, or `None` when the track ends. - /// The returned delivery carries the container's continuity through codec buffering. /// /// [`Frame::activity`] reports whether the packet these samples came from /// coded audio. It describes where the frame begins, so a resampled @@ -265,23 +263,35 @@ impl Consumer { /// side stay anchored to their own packet timeline, so the hole is there to /// see. "Doesn't continue" allows for the quantization the stamps carry, which /// on a millisecond-stamped ingest is most of a millisecond. - pub async fn read(&mut self) -> Result>, Error> { + pub async fn read(&mut self) -> Result, Error> { loop { if let Some(frame) = self.ready.pop_front() { return Ok(Some(frame)); } - let mux_frame = self.track.read().await?; - self.apply_discontinuity()?; - let Some(delivery) = mux_frame else { - if let Some(frame) = self.flush()? { - let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Delivery { frame, continuous }); + let mux_frame = match self.track.read().await? { + Some(Read::Frame { group, frame }) => { + // A later group after a declared end comes from a fresh encoder; the end's + // own group carries the stopped encoder's final packets, trimmed below. + if self.end.is_some() && group != self.end_group { + self.restart()?; + } + frame + } + Some(Read::Gap) => { + self.restart()?; + continue; + } + Some(Read::End { group, end }) => { + self.end = Some(end); + self.end_group = group; + self.frames_decoded = 0; + self.terminal_start = None; + continue; } - return Ok(self.ready.pop_front()); + None => return self.flush(), }; - let mux_frame = delivery.frame; // Every packet the container hands over is an arrival, including one the // decoder is about to refuse, so it is observed before anything can fail. let arrival = jitter::millis(now().duration_since(self.origin)); @@ -289,14 +299,6 @@ impl Consumer { let media = timestamp.value() as f64 * 1000.0 / timestamp.scale().as_u64() as f64; self.target.observe(arrival, media); - if let Some(end) = self.track.end() - && self.end != Some(end) - { - self.end = Some(end); - self.frames_decoded = 0; - self.terminal_start = None; - } - // Undeclared holes are routine: a skipped stalled group, a packet the // decoder refused, an ingest that resynced. Drop every stage's state at // the edge, before the packet after it goes anywhere near the decoder. @@ -310,11 +312,9 @@ impl Consumer { .is_some_and(|next| discontinuous(next, mux_frame.timestamp)) && let Some(frame) = self.gap()? { - let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Delivery { frame, continuous }); + self.ready.push_back(frame); } - self.continuous &= delivery.continuous; let rate = self.decoder.sample_rate(); let epoch = *self.epoch.get_or_insert(mux_frame.timestamp); let delay = self.decoder.delay_remaining(); @@ -408,26 +408,19 @@ impl Consumer { // Queued rather than returned, so a tail drained at a gap earlier in this // same iteration still comes out first. The next turn of the loop pops it. let frame = self.frame(pcm, timestamp, activity)?; - let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Delivery { frame, continuous }); + self.ready.push_back(frame); } } - /// A playhead event re-applies startup delay and skip. The decoder is not reset: + /// Start a new timeline after a gap or past a declared end. The decoder is not reset: /// the next group already starts on a keyframe, and pre-skip is a play-path concern. - fn apply_discontinuity(&mut self) -> Result<(), Error> { - let discontinuity = self.track.discontinuity(); - if discontinuity == self.discontinuity { - return Ok(()); - } - - // Samples already buffered belong to the old span, even when the next - // packet declares a new one. Drain before clearing their timeline labels. + /// + /// Samples the resampler still holds arrived before the break, so they are drained + /// as their own frame first rather than filtered together with what follows it. + fn restart(&mut self) -> Result<(), Error> { if let Some(frame) = self.drain()? { - let continuous = std::mem::replace(&mut self.continuous, true); - self.ready.push_back(Delivery { frame, continuous }); + self.ready.push_back(frame); } - self.discontinuity = discontinuity; self.next_start = None; self.spans.clear(); self.trailing = Activity::Active; @@ -461,7 +454,7 @@ impl Consumer { Ok(frame) } - /// Drain one span while keeping the resampler available for the next. + /// Drain what the resampler holds while keeping it for what follows. fn drain(&mut self) -> Result, Error> { let Some(resampler) = self.resampler.as_mut() else { return Ok(None); @@ -618,15 +611,16 @@ mod tests { use crate::encode::{Encoder, Input, Options as EncodeOptions, Producer, Settings}; use crate::{Format, Layout}; - #[tokio::test] - async fn continuity_survives_audio_decode_and_a_marker() { - use crate::decode::decoder::tests::{opus_catalog, opus_packets}; + /// What the resampler holds at a marker belongs before it: draining it after the next + /// packet would splice it onto the resumed audio and stamp that late. + #[tokio::test(start_paused = true)] + async fn resampled_tail_stays_before_a_marker() { let broadcast = moq_net::broadcast::Info::new().produce(); let track = broadcast .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) .unwrap(); let subscriber = broadcast.consume(); - let catalog = opus_catalog(moq_mux::codec::opus::Config::new(48_000, 1).with_pre_skip(312)); + let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 44_100, 1); let mut producer = moq_mux::container::Producer::new( track, moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), @@ -636,31 +630,40 @@ mod tests { &catalog, "audio", Options { - max_delay: std::time::Duration::from_secs(10), + max_delay: Duration::from_secs(10), + output: Output { + sample_rate: Some(48_000), + ..Output::default() + }, ..Options::new() }, ) .await .unwrap(); - for (index, payload) in opus_packets(4).into_iter().enumerate() { - if index == 2 { + for index in 0..2 { + if index != 0 { producer.discontinuity().unwrap(); } producer .write(moq_mux::container::Frame { - timestamp: Timestamp::from_micros(index as u64 * 20_000).unwrap(), + timestamp: Timestamp::from_micros(index * 1_000_000).unwrap(), duration: None, - payload, - keyframe: index % 2 == 0, + payload: vec![0u8; 1024 * size_of::()].into(), + keyframe: true, }) .unwrap(); } producer.finish().unwrap(); - let mut flags = Vec::new(); - while let Some(delivery) = consumer.read().await.unwrap() { - flags.push(delivery.continuous); + let mut stamps = Vec::new(); + while let Some(frame) = consumer.read().await.unwrap() { + stamps.push(frame.timestamp.as_micros()); } - assert_eq!(flags, [false, true, false, true]); + assert_eq!(stamps.len(), 4, "each span's body and tail: {stamps:?}"); + assert!( + stamps[1] < 1_000_000, + "the old tail stays before the marker: {stamps:?}" + ); + assert_eq!(stamps[2], 1_000_000, "the resumed span starts on its source timestamp"); } #[tokio::test] @@ -705,7 +708,7 @@ mod tests { producer.write(&Frame::new(data.into(), Timestamp::ZERO)).unwrap(); let frame = consumer.read().await.unwrap().expect("decoded frame"); - let samples = Format::F32.as_interleaved_f32(&frame.frame.data, 2).unwrap(); + let samples = Format::F32.as_interleaved_f32(&frame.data, 2).unwrap(); assert_eq!(samples.len(), (960 - 312) * 2); for pair in samples.as_chunks::<2>().0.iter() { assert_eq!(pair[0], pair[1]); @@ -749,8 +752,8 @@ mod tests { // 312 samples at 48 kHz is 6.5 ms. for (micros, frames) in [(0, 960 - 312), (13_500, 960), (33_500, 960)] { let frame = consumer.read().await.unwrap().expect("decoded frame"); - assert_eq!(frame.frame.timestamp.as_micros(), micros); - assert_eq!(frame.frame.data.len() / size_of::(), frames); + assert_eq!(frame.timestamp.as_micros(), micros); + assert_eq!(frame.data.len() / size_of::(), frames); } } @@ -805,7 +808,7 @@ mod tests { } let first = consumer.read().await.unwrap().expect("decoded frame"); - assert_eq!(first.frame.timestamp.as_micros(), 0); + assert_eq!(first.timestamp.as_micros(), 0); // Continuity, not a fixed number: the second frame starts where the first // one's samples end, whatever they came to. Within a few frames rather than @@ -813,9 +816,9 @@ mod tests { // wobbles around the nominal ratio; a real hole (the samples it held back, or // the startup silence it dropped) is twenty times this tolerance. let second = consumer.read().await.unwrap().expect("decoded frame"); - let first_frames = (first.frame.data.len() / size_of::()) as u128; + let first_frames = (first.data.len() / size_of::()) as u128; let ends_at = first_frames * 1_000_000 / 48_000; - let gap = second.frame.timestamp.as_micros().abs_diff(ends_at); + let gap = second.timestamp.as_micros().abs_diff(ends_at); assert!(gap < 100, "expected the frames to meet, got a {gap} us gap"); } @@ -865,10 +868,10 @@ mod tests { producer.finish().unwrap(); let first = consumer.read().await.unwrap().expect("decoded frame"); - let first_frames = first.frame.data.len() / size_of::(); + let first_frames = first.data.len() / size_of::(); let tail = consumer.read().await.unwrap().expect("flushed tail"); - let tail_frames = tail.frame.data.len() / size_of::(); + let tail_frames = tail.data.len() / size_of::(); // The 142 held-back frames at 44.1 kHz are ~155 at 48 kHz, plus the 69 the // sinc filter still owes: it runs centred, so the end of the track only @@ -877,7 +880,7 @@ mod tests { // It picks up where the first frame's samples ended, within the same few // frames of whole-frame rounding as above. let ends_at = (first_frames as u128) * 1_000_000 / 48_000; - let gap = tail.frame.timestamp.as_micros().abs_diff(ends_at); + let gap = tail.timestamp.as_micros().abs_diff(ends_at); assert!(gap < 100, "expected the tail to meet the body, got a {gap} us gap"); // Together they cover the packet and no more: 1024 frames at 44.1 kHz is @@ -888,62 +891,6 @@ mod tests { assert!(consumer.read().await.unwrap().is_none()); } - #[tokio::test(start_paused = true)] - async fn resampled_continuity_keeps_the_tail_before_a_marker() { - let broadcast = moq_net::broadcast::Info::new().produce(); - let track = broadcast - .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) - .unwrap(); - let subscriber = broadcast.consume(); - let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 44_100, 1); - let mut producer = moq_mux::container::Producer::new( - track, - moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), - ); - let mut consumer = Consumer::new( - &subscriber, - &catalog, - "audio", - Options { - max_delay: Duration::from_secs(10), - output: Output { - sample_rate: Some(48_000), - ..Output::default() - }, - ..Options::new() - }, - ) - .await - .unwrap(); - for index in 0..2 { - if index != 0 { - producer.discontinuity().unwrap(); - } - producer - .write(moq_mux::container::Frame { - timestamp: Timestamp::from_micros(index * 1_000_000).unwrap(), - duration: None, - payload: vec![0u8; 1024 * size_of::()].into(), - keyframe: true, - }) - .unwrap(); - } - producer.finish().unwrap(); - let mut samples = Vec::new(); - while let Some(delivery) = consumer.read().await.unwrap() { - samples.push((delivery.frame.timestamp.as_micros(), delivery.continuous)); - } - assert_eq!( - samples.iter().map(|(_, continuous)| *continuous).collect::>(), - [false, true, false, true] - ); - assert!( - samples[1].0 < 1_000_000, - "the old resampler tail stays before the marker" - ); - assert_eq!(samples[2].0, 1_000_000, "the new span starts on its source timestamp"); - } - #[tokio::test] async fn resampling_keeps_the_activity_boundary_on_its_source() { let mut encoder = Encoder::new(&Settings { @@ -1008,9 +955,9 @@ mod tests { // nothing every other packet. Those must not surface as frames: a frame // with no samples reads as audio arriving, and carries an activity // describing samples that are not there. - assert!(!frame.frame.data.is_empty(), "read returned a frame with no samples"); - if frame.frame.activity.is_dtx() { - actual = Some(frame.frame.timestamp); + assert!(!frame.data.is_empty(), "read returned a frame with no samples"); + if frame.activity.is_dtx() { + actual = Some(frame.timestamp); break; } } @@ -1078,10 +1025,7 @@ mod tests { let mut read = Vec::new(); while let Some(frame) = consumer.read().await.unwrap() { - read.push(( - frame.frame.timestamp.as_micros(), - frame.frame.data.len() / size_of::(), - )); + read.push((frame.timestamp.as_micros(), frame.data.len() / size_of::())); } read } @@ -1232,9 +1176,9 @@ mod tests { // The output begins with the first packet's samples, so it is stamped and // labelled from that packet. Rewinding from the terminal one instead drops it // most of a second into the future, carrying the DTX label with it. - assert_eq!(frame.frame.timestamp.as_micros(), 0, "held samples moved with the jump"); + assert_eq!(frame.timestamp.as_micros(), 0, "held samples moved with the jump"); assert!( - frame.frame.activity.is_active(), + frame.activity.is_active(), "held samples took the terminal packet's label" ); } @@ -1332,24 +1276,20 @@ mod tests { // shortfall is codec delay, not a hole: without counting it the packet after // every stream start would read as a gap. let first = consumer.read().await.unwrap().expect("decoded frame"); - let frames = first.frame.data.len() / size_of::(); + let frames = first.data.len() / size_of::(); assert!(frames < 960, "the pre-skip should be trimmed, got {frames} frames"); // The hole is real, so codec prediction starts over but stream-level pre-skip // does not. The audio is stamped where the packet says rather than 2.5 ms early. let second = consumer.read().await.unwrap().expect("decoded frame"); - assert_eq!(second.frame.timestamp.as_micros(), 22_500); - assert_eq!( - second.frame.data.len() / size_of::(), - 960, - "pre-skip was reapplied" - ); + assert_eq!(second.timestamp.as_micros(), 22_500); + assert_eq!(second.data.len() / size_of::(), 960, "pre-skip was reapplied"); let third = consumer.read().await.unwrap().expect("decoded frame after gap"); - let second_frames = second.frame.data.len() / size_of::(); + let second_frames = second.data.len() / size_of::(); assert_eq!( - third.frame.timestamp, - advance(second.frame.timestamp, second_frames, 48_000).unwrap() + third.timestamp, + advance(second.timestamp, second_frames, 48_000).unwrap() ); } @@ -1482,7 +1422,7 @@ mod tests { let frame = consumer.read().await.unwrap().expect("decoded frame"); assert!( - frame.frame.data.len() / size_of::() < encoder.frame_size(), + frame.data.len() / size_of::() < encoder.frame_size(), "pre-skip is trimmed" ); // The cold-start prior reads 100 ms, plus one 20 ms packet. @@ -1587,9 +1527,9 @@ mod tests { let first = consumer.read().await.unwrap().expect("first decoded frame"); let second = consumer.read().await.unwrap().expect("second decoded frame"); - let first_frames = first.frame.data.len() / size_of::(); - let expected = advance(first.frame.timestamp, first_frames, 48_000).unwrap(); - assert_eq!(second.frame.timestamp, expected); + let first_frames = first.data.len() / size_of::(); + let expected = advance(first.timestamp, first_frames, 48_000).unwrap(); + assert_eq!(second.timestamp, expected); } #[tokio::test] @@ -1645,8 +1585,8 @@ mod tests { let first = consumer.read().await.unwrap().expect("first decoded frame"); let _second = consumer.read().await.unwrap().expect("second decoded frame"); let resumed = consumer.read().await.unwrap().expect("resumed decoded frame"); - let first_frames = first.frame.data.len() / size_of::(); - let resumed_frames = resumed.frame.data.len() / size_of::(); + let first_frames = first.data.len() / size_of::(); + let resumed_frames = resumed.data.len() / size_of::(); assert!(first_frames < frame_size, "the first epoch trims pre-skip"); assert_eq!( resumed_frames, first_frames, @@ -1697,7 +1637,7 @@ mod tests { let frame = consumer.read().await.unwrap().expect("decoded frame"); assert_eq!( - Format::F32.as_interleaved_f32(&frame.frame.data, 1).unwrap().as_ref(), + Format::F32.as_interleaved_f32(&frame.data, 1).unwrap().as_ref(), samples ); } @@ -1753,8 +1693,8 @@ mod tests { let frame = consumer.read().await.unwrap().expect("decoded frame"); // `as_micros`, not `==`: the CMAF path carries the fmp4 timescale and // `Timestamp`'s equality is structural, so the scales would have to match too. - assert_eq!(frame.frame.timestamp.as_micros(), 0); - let samples = Format::F32.as_interleaved_f32(&frame.frame.data, 2).unwrap(); + assert_eq!(frame.timestamp.as_micros(), 0); + let samples = Format::F32.as_interleaved_f32(&frame.data, 2).unwrap(); assert_eq!(samples.len(), (960 - 312) * 2); } } diff --git a/rs/moq-audio/src/encode/capture.rs b/rs/moq-audio/src/encode/capture.rs index a9462765b4b..eb4f92b9ea5 100644 --- a/rs/moq-audio/src/encode/capture.rs +++ b/rs/moq-audio/src/encode/capture.rs @@ -2193,8 +2193,11 @@ mod tests { async fn read_new(&self, track: &mut Track, seen: &[u64]) -> u64 { loop { - let packet = track.read().await.unwrap().expect("a published packet"); - let timestamp = u64::try_from(packet.frame.timestamp.as_micros()).unwrap(); + let Some(moq_mux::container::Read::Frame { frame: packet, .. }) = track.read().await.unwrap() + else { + continue; + }; + let timestamp = u64::try_from(packet.timestamp.as_micros()).unwrap(); if !seen.contains(×tamp) { return timestamp; } diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index 456b4601522..bc68c6a7034 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -565,6 +565,18 @@ mod tests { use crate::decode::{Consumer as AudioConsumer, Options as DecodeOptions}; use crate::{Activity, Format, Layout}; + /// The next published packet, passing over gaps and ends. + async fn packet(reader: &mut moq_mux::container::Consumer) -> Option + where + F::Error: std::fmt::Debug, + { + loop { + if let moq_mux::container::Read::Frame { frame, .. } = reader.read().await.unwrap()? { + return Some(frame); + } + } + } + #[tokio::test] async fn demand_follows_subscribers_and_closes_with_the_producer() { let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -648,7 +660,7 @@ mod tests { let mut decoded = Vec::new(); while let Some(frame) = audio.read().await.unwrap() { - let pcm = Format::F32.as_interleaved_f32(&frame.frame.data, 1).unwrap(); + let pcm = Format::F32.as_interleaved_f32(&frame.data, 1).unwrap(); decoded.extend_from_slice(&pcm); } assert_eq!(decoded.len(), frames, "terminal padding extended the source"); @@ -699,7 +711,7 @@ mod tests { producer.finish().unwrap(); let mut packets = 0; - while track.read().await.unwrap().is_some() { + while packet(&mut track).await.is_some() { packets += 1; } assert_eq!(packets, 11); @@ -782,8 +794,8 @@ mod tests { producer.reanchor(); producer.finish().unwrap(); - assert!(track.read().await.unwrap().is_some()); - assert!(track.read().await.unwrap().is_none()); + assert!(packet(&mut track).await.is_some()); + assert!(packet(&mut track).await.is_none()); } /// A codec reset starts a new pre-skip interval at the receiver too. @@ -819,7 +831,7 @@ mod tests { producer.write(&full_frame(0)).unwrap(); let first = audio.read().await.unwrap().expect("first packet before the reanchor"); - assert_eq!(first.frame.data.len() / size_of::(), 960 - 312); + assert_eq!(first.data.len() / size_of::(), 960 - 312); producer.reanchor(); producer.write(&full_frame(1_000_000)).unwrap(); @@ -827,8 +839,8 @@ mod tests { let mut resumed_frames = 0; while let Some(frame) = audio.read().await.unwrap() { - if frame.frame.timestamp.as_micros() >= 1_000_000 { - resumed_frames += frame.frame.data.len() / size_of::(); + if frame.timestamp.as_micros() >= 1_000_000 { + resumed_frames += frame.data.len() / size_of::(); } } assert!(resumed_frames > 0, "the reanchored audio still decodes"); @@ -866,8 +878,8 @@ mod tests { for index in 0..100 { producer.write(&pcm_frame(&silence, index * 20_000)).unwrap(); let consumed = consumer.read().await.unwrap().expect("one decoded frame"); - assert_eq!(producer.activity(), consumed.frame.activity); - if consumed.frame.activity.is_dtx() { + assert_eq!(producer.activity(), consumed.activity); + if consumed.activity.is_dtx() { entered_dtx = true; break; } @@ -883,7 +895,7 @@ mod tests { producer.write(&pcm_frame(&active, 2_000_000)).unwrap(); let consumed = consumer.read().await.unwrap().expect("one decoded frame"); assert_eq!(producer.activity(), Activity::Active); - assert_eq!(consumed.frame.activity, Activity::Active); + assert_eq!(consumed.activity, Activity::Active); } // One 20 ms Opus frame at 48 kHz mono is exactly 960 f32 samples, so each @@ -928,8 +940,8 @@ mod tests { producer.reanchor(); } producer.write(frame).unwrap(); - let read = reader.read().await.unwrap().expect("a packet per full frame"); - pts.push(read.frame.timestamp.as_micros()); + let read = packet(&mut reader).await.expect("a packet per full frame"); + pts.push(read.timestamp.as_micros()); } pts } @@ -1118,16 +1130,7 @@ mod tests { producer.write(&pcm_frame(&[0.1; 4 * 1024], anchor_us)).unwrap(); let mut pts = Vec::new(); for _ in 0..4 { - pts.push( - reader - .read() - .await - .unwrap() - .expect("a packet") - .frame - .timestamp - .as_micros(), - ); + pts.push(packet(&mut reader).await.expect("a packet").timestamp.as_micros()); } pts } @@ -1176,8 +1179,8 @@ mod tests { // A second of audio, so the filter's delay is nowhere near the whole write. producer.write(&pcm_frame(&vec![0.1; 44_100], 1_000_000)).unwrap(); - let first = reader.read().await.unwrap().expect("a packet"); - assert_eq!(first.frame.timestamp.as_micros(), 1_000_000); + let first = packet(&mut reader).await.expect("a packet"); + assert_eq!(first.timestamp.as_micros(), 1_000_000); } #[tokio::test] @@ -1228,7 +1231,7 @@ mod tests { producer.write(&full_frame(0)).unwrap(); producer.finish().unwrap(); - assert!(track.read().await.unwrap().is_some()); + assert!(packet(&mut track).await.is_some()); producer.abort(moq_net::Error::Cancel); } diff --git a/rs/moq-audio/src/playback.rs b/rs/moq-audio/src/playback.rs index 884a06a48cf..798bb5984dc 100644 --- a/rs/moq-audio/src/playback.rs +++ b/rs/moq-audio/src/playback.rs @@ -18,7 +18,7 @@ //! let mut sink = engine.sink(input)?; //! //! while let Some(frame) = audio.read().await? { -//! let write = sink.write(&frame.frame.data)?; +//! let write = sink.write(&frame.data)?; //! if write.dropped_sample_frames > 0 { //! eprintln!("dropped {} live audio frames", write.dropped_sample_frames); //! } diff --git a/rs/moq-audio/tests/import.rs b/rs/moq-audio/tests/import.rs index 160f75faeee..a1a5a147e43 100644 --- a/rs/moq-audio/tests/import.rs +++ b/rs/moq-audio/tests/import.rs @@ -39,7 +39,7 @@ async fn ts_surround_opus_decodes() { .expect("decoded frame timed out") .unwrap() { - let (samples, rest) = frame.frame.data.as_chunks::<4>(); + let (samples, rest) = frame.data.as_chunks::<4>(); assert!(rest.is_empty()); for chunk in samples.as_chunks::<6>().0 { for (channel, sample) in energy.iter_mut().zip(chunk) { diff --git a/rs/moq-audio/tests/roundtrip.rs b/rs/moq-audio/tests/roundtrip.rs index af63857020c..34ba2750638 100644 --- a/rs/moq-audio/tests/roundtrip.rs +++ b/rs/moq-audio/tests/roundtrip.rs @@ -76,7 +76,6 @@ async fn opus_round_trip_48k_stereo() { .unwrap() { let pcm: Vec = frame - .frame .data .as_chunks::<4>() .0 @@ -152,7 +151,7 @@ async fn opus_round_trip_44100_s16_resampled() { .expect("decoded frame timed out") .unwrap() { - total_bytes += frame.frame.data.len() as u64; + total_bytes += frame.data.len() as u64; } assert!( total_bytes > 10_000, @@ -193,9 +192,8 @@ async fn pcm_round_trip_is_lossless() { producer.finish().unwrap(); let frame = consumer.read().await.unwrap().unwrap(); - assert_eq!(frame.frame.timestamp, Timestamp::from_micros(123_000).unwrap()); + assert_eq!(frame.timestamp, Timestamp::from_micros(123_000).unwrap()); let decoded: Vec = frame - .frame .data .as_chunks::<4>() .0 @@ -261,7 +259,6 @@ async fn pcm_five_one_reads_back_at_any_layout() { assert_eq!(consumer.layout(), layout); let decoded = consumer.read().await.unwrap().unwrap(); let samples: Vec = decoded - .frame .data .as_chunks::<4>() .0 diff --git a/rs/moq-c/src/audio.rs b/rs/moq-c/src/audio.rs index 5bd108953ba..f13b3991ba0 100644 --- a/rs/moq-c/src/audio.rs +++ b/rs/moq-c/src/audio.rs @@ -272,7 +272,7 @@ impl Audio { }; // Hold the lock only to buffer the frame; release it before the callback. - let frame_id = State::lock().audio.frames.insert(frame.frame)?; + let frame_id = State::lock().audio.frames.insert(frame)?; callback.call(Ok(frame_id)); } } diff --git a/rs/moq-c/src/consume.rs b/rs/moq-c/src/consume.rs index 66c71f2233b..8c0efdfb479 100644 --- a/rs/moq-c/src/consume.rs +++ b/rs/moq-c/src/consume.rs @@ -539,14 +539,16 @@ impl Consume { let frame = tokio::select! { biased; _ = &mut close => return Ok(()), - frame = track.read() => match frame? { - Some(frame) => frame, + read = track.read() => match read? { + Some(moq_mux::container::Read::Frame { frame, .. }) => frame, + // The C ABI carries frames only. + Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. }) => continue, None => return Ok(()), }, }; // Hold the lock only to buffer the frame; release it before the callback. - let frame_id = State::lock().consume.frame.insert(frame.frame)?; + let frame_id = State::lock().consume.frame.insert(frame)?; callback.call(Ok(frame_id)); } } diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index 9ba35c89676..8676e459428 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -442,7 +442,7 @@ async fn play_audio( loop { let frame = match consumer.read().await { - Ok(Some(delivery)) => delivery.frame, + Ok(Some(frame)) => frame, Ok(None) => break, // One bad packet is that packet's problem: the decoder stays usable, so // skip it rather than ending playback and taking the video window down diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index a97506f3600..890b1c628e3 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeSet, VecDeque}; use std::sync::{Arc, Mutex}; use std::time::Duration; -use moq_mux::container::Frame; +use moq_mux::container::{Frame, Read}; use tokio::time::Instant; use super::buffer::Buffer; @@ -73,31 +73,34 @@ impl Video { pub async fn run(self, mut track: Track, mut decoder: impl Decoder) -> anyhow::Result<()> { let buffer = Mutex::new(Buffer::default()); let receive = async { - let mut discontinuity = track.discontinuity(); loop { - let frame = track.read().await?; + let read = track.read().await?; let mut buffer = buffer.lock().unwrap(); let before = buffer.generation; - if track.discontinuity() != discontinuity { - discontinuity = track.discontinuity(); - buffer.clear(); - self.presentation.lock().unwrap().video_restarted(); - } - let ended = frame.is_none(); - if let Some(delivery) = frame { - let frame = delivery.frame; - if self - .presentation - .lock() - .unwrap() - .video(frame.timestamp, Instant::now().into_std()) - { - self.output.send(Event::Wake); + let ended = match read { + Some(Read::Frame { frame, .. }) => { + if self + .presentation + .lock() + .unwrap() + .video(frame.timestamp, Instant::now().into_std()) + { + self.output.send(Event::Wake); + } + buffer.push(frame, self.max_delay); + false } - buffer.push(frame, self.max_delay); - } else { - buffer.ended = true; - } + // Applied in delivery order, so the restart lands before the frame after it. + Some(Read::Gap | Read::End { .. }) => { + buffer.clear(); + self.presentation.lock().unwrap().video_restarted(); + false + } + None => { + buffer.ended = true; + true + } + }; if buffer.generation != before { self.frames.lock().unwrap().clear(); self.output.send(Event::Wake); diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index 82786c4f3b8..ec1c356720b 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -1173,13 +1173,7 @@ pub(crate) mod tests { let (name, config) = catalog.video.renditions.iter().next().expect("a video rendition"); let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let container = moq_mux::catalog::hang::Container::try_from(config).unwrap(); - let first = Consumer::new(track, container) - .read() - .await - .unwrap() - .expect("a video frame") - .frame - .timestamp; + let first = next_frame(&mut Consumer::new(track, container)).await.timestamp; assert!( first.as_micros() < Duration::from_secs(5).as_micros(), "the feed's own PTS: {first:?}" @@ -1455,11 +1449,22 @@ pub(crate) mod tests { async fn read_frame(consumer: &moq_net::broadcast::Consumer, name: &str) -> Vec { let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let mut reader = Consumer::new(track, Container::Legacy(moq_mux::container::Kind::Data)); - let frame = tokio::time::timeout(Duration::from_secs(1), reader.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), next_frame(&mut reader)) .await - .expect("verbatim read timed out") - .unwrap() - .expect("a published verbatim frame"); - frame.frame.payload.to_vec() + .expect("verbatim read timed out"); + frame.payload.to_vec() + } + + /// The next frame, passing over gaps and ends. + async fn next_frame(reader: &mut Consumer) -> moq_mux::container::Frame + where + C::Error: std::fmt::Debug, + { + loop { + let read = reader.read().await.unwrap().expect("a published frame"); + if let moq_mux::container::Read::Frame { frame, .. } = read { + return frame; + } + } } } diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index 5d3463a85a8..172db6ecc93 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -342,7 +342,7 @@ struct ConsumerInner { impl ConsumerInner { async fn next(&mut self) -> Result, MoqError> { - Ok(self.consumer.read().await?.map(|delivery| delivery.frame.into())) + Ok(self.consumer.read().await?.map(Into::into)) } } diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 0c8fc417b17..753745fc152 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -1101,11 +1101,14 @@ struct Media { impl Media { async fn next(&mut self) -> Result, MoqError> { - self.inner - .read() - .await? - .map(|delivery| media_frame(delivery.frame)) - .transpose() + loop { + match self.inner.read().await? { + Some(moq_mux::container::Read::Frame { frame, .. }) => return media_frame(frame).map(Some), + // The bindings carry frames only. + Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. }) => continue, + None => return Ok(None), + } + } } } diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index c8d05447892..9b9bb29ca6d 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1094,7 +1094,8 @@ impl Pump { // This rendition is being torn down (its run ended, or a catalog update replaced it). _ = cancel.changed() => return None, frame = track.read() => match frame { - Ok(Some(frame)) => { + Ok(Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. })) => {} + Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { // PTS restarts at zero with every pump, so the segment places that zero at the // current running time, or a synced sink would drop a restarted run as late. let segment = reference_ts.is_none().then(|| { @@ -1107,7 +1108,7 @@ impl Pump { ); gst::event::Segment::new(&segment) }); - let buffer = build_buffer(frame.frame, &mut reference_ts, kind); + let buffer = build_buffer(frame, &mut reference_ts, kind); // pad.push() blocks until downstream accepts the buffer (full queues, a // clock-synced sink). block_in_place hands our sibling tasks to another // worker so a stalled downstream can't pin a runtime thread and starve diff --git a/rs/moq-mux/src/codec/opus/import.rs b/rs/moq-mux/src/codec/opus/import.rs index ed290b67831..1abbbed143e 100644 --- a/rs/moq-mux/src/codec/opus/import.rs +++ b/rs/moq-mux/src/codec/opus/import.rs @@ -173,13 +173,13 @@ mod tests { subscriber, crate::catalog::hang::Container::Loc(crate::container::Kind::Data), ); - let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.frame.payload.as_ref(), payload); - assert_eq!(frame.frame.timestamp, Timestamp::from_micros(1_000).unwrap()); + assert_eq!(frame.payload.as_ref(), payload); + assert_eq!(frame.timestamp, Timestamp::from_micros(1_000).unwrap()); import.finish().unwrap(); } diff --git a/rs/moq-mux/src/codec/vp8/import.rs b/rs/moq-mux/src/codec/vp8/import.rs index f8cb4ce667b..0d6321b68a4 100644 --- a/rs/moq-mux/src/codec/vp8/import.rs +++ b/rs/moq-mux/src/codec/vp8/import.rs @@ -206,13 +206,13 @@ mod tests { subscriber, crate::catalog::hang::Container::Loc(crate::container::Kind::Data), ); - let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.frame.payload, keyframe); - assert_eq!(frame.frame.timestamp, Timestamp::from_micros(0).unwrap()); + assert_eq!(frame.payload, keyframe); + assert_eq!(frame.timestamp, Timestamp::from_micros(0).unwrap()); import.finish().unwrap(); } diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 42f4703bc86..51102c62051 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -3,12 +3,17 @@ use std::task::{Poll, ready}; use moq_net::Timestamp; -use super::{Container, Delivery, Frame, TimestampRewind}; +use super::{Container, Frame, Read, TimestampRewind}; /// Media and clean group boundaries in delivery order. pub(crate) enum Event { Frame(Frame), - FrameEnd(Timestamp), + /// An endpoint marker. `opens` when no media precedes it in its group: a declared stop + /// (a marker group or an audio terminal) rather than the previous frame's duration. + FrameEnd { + end: Timestamp, + opens: bool, + }, GroupEnd, } @@ -49,13 +54,12 @@ pub(crate) enum Event { /// subscribing. [`new`](Self::new) inherits that budget, and /// [`set_max_delay`](Self::set_max_delay) changes it mid-stream. /// -/// ## Timeline discontinuities +/// ## Gaps and ends /// -/// A marker group (no decodable frames, one empty payload) is a walk-now discontinuity. -/// A delivered sequence hole is a playhead event unless the boundary is contiguous within -/// 1 ms, and a latency skip or an evicted current group is the same event. -/// [`discontinuity`](Self::discontinuity) is that playhead generation: re-apply startup -/// delay and skip, not a decoder flush. +/// [`read`](Self::read) yields [`Read::Gap`] before the first frame and wherever media +/// will never arrive: a latency skip, an evicted current group, or a sequence hole whose +/// boundary is not contiguous within 1 ms. A marker group (no decodable frames, one empty +/// payload) is the publisher declaring a stop, read as [`Read::End`]; nothing is missing. /// Empty groups (zero objects) mean nothing. A group whose timestamps fall below the live /// edge earlier groups reached is malformed and aborts the track. pub struct Consumer { @@ -95,13 +99,19 @@ pub struct Consumer { // Increments on a declared marker, an unproven delivered hole, and a latency skip. discontinuity: u64, - delivered: Option, // Increments on a declared marker only: the publisher paused or broke its timeline forward. markers: u64, - // Exclusive audio endpoint delivered before terminal codec packets. - end: Option, + // Media went missing since the last item read, so the next one is preceded by a gap. + // Starts set: nothing before the first frame was read. + gap: bool, + + // The item a gap was returned ahead of. + next: Option, + + // The group the last event came from. + group: u64, } /// Two adjacent groups are timeline-contiguous when the next start is within this slack @@ -145,18 +155,16 @@ impl Consumer { floor: None, presented_end: None, discontinuity: 0, - delivered: None, markers: 0, - end: None, + gap: true, + next: None, + group: 0, } } /// A counter that increments at each playhead event: a declared marker group, an /// unproven delivered hole, a latency skip, or an evicted current group. - /// - /// Downstream consumers re-apply startup delay and skip when it changes. It is not a - /// decoder flush: the next group already starts on a keyframe with parameter sets. - pub fn discontinuity(&self) -> u64 { + pub(crate) fn discontinuity(&self) -> u64 { self.discontinuity } @@ -168,44 +176,54 @@ impl Consumer { self.markers } - /// The exclusive audio endpoint delivered before terminal codec packets. - pub fn end(&self) -> Option { - self.end - } - /// Read the next frame from the track. /// /// This method handles timestamp decoding, group ordering, and age management /// automatically. It will skip groups that are too far behind to maintain the /// configured max delay. /// - /// Returns `None` when the track has ended. - /// The first delivery and the first one after a playhead event have `continuous: false`. - /// An undeclared forward timestamp jump alone does not change the flag. - pub async fn read(&mut self) -> Result, F::Error> { + /// Returns `None` when the track has ended. See [`Read`] for the items between frames; + /// an undeclared forward timestamp jump is not a gap, since nothing says media is missing. + pub async fn read(&mut self) -> Result, F::Error> { kio::wait(|waiter| self.poll_read(waiter)).await } + /// The next frame, passing over gaps and ends. + #[cfg(test)] + pub(crate) async fn read_frame(&mut self) -> Result, F::Error> { + loop { + match self.read().await? { + Some(Read::Frame { frame, .. }) => return Ok(Some(frame)), + Some(Read::Gap | Read::End { .. }) => continue, + None => return Ok(None), + } + } + } + /// Poll-based implementation of the read loop. /// /// Uses a single waiter that gets registered on all relevant kio channels, /// avoiding the need for `tokio::select!` or `FuturesUnordered`. - pub fn poll_read(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { + pub fn poll_read(&mut self, waiter: &kio::Waiter) -> Poll, F::Error>> { + if let Some(read) = self.next.take() { + return Poll::Ready(Ok(Some(read))); + } loop { - match ready!(self.poll_event(waiter))? { - Some(Event::Frame(frame)) => { - let continuous = self.delivered == Some(self.discontinuity); - self.delivered = Some(self.discontinuity); - return Poll::Ready(Ok(Some(Delivery { frame, continuous }))); - } - Some(Event::FrameEnd(end)) => { - if self.format.kind() == super::Kind::Audio { - self.end = Some(end); - } - } - Some(Event::GroupEnd) => continue, + let read = match ready!(self.poll_event(waiter))? { + Some(Event::Frame(frame)) => Read::Frame { + group: self.group, + frame, + }, + Some(Event::FrameEnd { end, opens: true }) => Read::End { group: self.group, end }, + // A trailing endpoint is the last frame's duration, which only the skip logic needs. + Some(Event::FrameEnd { opens: false, .. } | Event::GroupEnd) => continue, None => return Poll::Ready(Ok(None)), + }; + if std::mem::take(&mut self.gap) { + self.next = Some(read); + return Poll::Ready(Ok(Some(Read::Gap))); } + return Poll::Ready(Ok(Some(read))); } } @@ -266,9 +284,10 @@ impl Consumer { if self.live_edge.is_none_or(|(_, high)| ts.as_micros() > high.as_micros()) { self.live_edge = Some((seq, ts)); } + self.group = seq; return Poll::Ready(Ok(Some(Event::Frame(frame)))); } - Poll::Ready(Ok(Some(Event::FrameEnd(end)))) => { + Poll::Ready(Ok(Some(Event::FrameEnd { end, opens }))) => { let seq = group.group.sequence; if self .live_edge @@ -276,7 +295,8 @@ impl Consumer { { self.live_edge = Some((seq, end)); } - return Poll::Ready(Ok(Some(Event::FrameEnd(end)))); + self.group = seq; + return Poll::Ready(Ok(Some(Event::FrameEnd { end, opens }))); } Poll::Ready(Ok(Some(event))) => return Poll::Ready(Ok(Some(event))), // Still blocked on this group, don't skip it yet. @@ -309,7 +329,7 @@ impl Consumer { "current group evicted; skipping to next buffered group" ); // Whatever the group still held is lost, so this is a skip like any other. - self.bump_playhead(); + self.lose(); self.pending.pop_front(); self.current = self.pending.front().map_or(self.current + 1, |g| g.sequence); continue 'read; @@ -386,7 +406,7 @@ impl Consumer { && (finished || max_timestamp.saturating_sub(next_start) >= self.max_delay) { if !pts_contiguous(current_end.or(self.presented_end), next_start) { - self.bump_playhead(); + self.lose(); } self.current = front_sequence; self.note_group_edge(); @@ -411,9 +431,15 @@ impl Consumer { { let hole = !pts_contiguous(current_end.or(self.presented_end), next_start); let had_marker = self.pending.iter().take(new_idx).any(GroupBuffer::marker); + // Buffered frames are thrown away, and `current_end` counts them, so contiguity alone would hide them. + let discarded = self + .pending + .iter() + .take(new_idx) + .any(|group| !group.buffered.is_empty()); self.pending.drain(0..new_idx); - if hole || had_marker { - self.bump_playhead(); + if hole || had_marker || discarded { + self.lose(); } self.markers += u64::from(had_marker); let new_current = self.pending.front().map(|g| g.sequence).unwrap(); @@ -437,7 +463,7 @@ impl Consumer { if let Some(start) = next_start && !pts_contiguous(current_end.or(self.presented_end), start) { - self.bump_playhead(); + self.lose(); } self.current = front_sequence; self.note_group_edge(); @@ -454,7 +480,12 @@ impl Consumer { fn bump_playhead(&mut self) { self.discontinuity += 1; - self.end = None; + } + + // Media this reader will never deliver: the next item is preceded by a gap. + fn lose(&mut self) { + self.bump_playhead(); + self.gap = true; } fn note_group_edge(&mut self) { @@ -586,8 +617,8 @@ impl GroupBuffer { fn poll_read(&mut self, waiter: &kio::Waiter, format: &F) -> Poll, F::Error>> { loop { if self.markers.front().is_some_and(|(index, _)| *index <= self.delivered) { - let (_, end) = self.markers.pop_front().unwrap(); - return Poll::Ready(Ok(Some(Event::FrameEnd(end)))); + let (index, end) = self.markers.pop_front().unwrap(); + return Poll::Ready(Ok(Some(Event::FrameEnd { end, opens: index == 0 }))); } if let Some(frame) = self.buffered.pop_front() { self.delivered += 1; @@ -837,8 +868,8 @@ mod tests { async fn read_all(consumer: &mut Consumer) -> Result, crate::Error> { let mut frames = Vec::new(); loop { - match tokio::time::timeout(Duration::from_millis(200), consumer.read()).await { - Ok(Ok(Some(frame))) => frames.push(frame.frame), + match tokio::time::timeout(Duration::from_millis(200), consumer.read_frame()).await { + Ok(Ok(Some(frame))) => frames.push(frame), Ok(Ok(None)) => break, Ok(Err(e)) => return Err(e), Err(_) => panic!( @@ -919,27 +950,49 @@ mod tests { group.finish().unwrap(); } + /// Every item until the track ends: a frame as its timestamp in µs, a gap, or an end. + async fn items(consumer: &mut Consumer) -> Vec { + let mut items = Vec::new(); + while let Some(read) = consumer.read().await.unwrap() { + items.push(match read { + Read::Frame { frame, .. } => frame.timestamp.as_micros().to_string(), + Read::Gap => "gap".to_string(), + Read::End { end, .. } => format!("end {}", end.as_micros()), + }); + } + items + } + #[tokio::test] - async fn continuity_marks_subscribe_holes_and_markers() { - let mut track = track_producer("continuity", hang::container::track_info(hang::catalog::PRIORITY.audio)); + async fn gaps_mark_the_join_and_holes_while_a_marker_reads_as_an_end() { + let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); let subscription = track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); write_group(&mut track, 0, &[ts(0), ts(10_000)]); write_group(&mut track, 2, &[ts(1_000_000), ts(1_010_000)]); - write_marker_group(&mut track, 3, ts(1_010_000)); + write_marker_group(&mut track, 3, ts(1_020_000)); write_group(&mut track, 4, &[ts(2_000_000), ts(2_010_000)]); track.finish().unwrap(); - let mut flags = Vec::new(); - while let Some(delivery) = reader.read().await.unwrap() { - flags.push(delivery.continuous); - } - assert_eq!(flags, [false, true, false, true, false, true]); + assert_eq!( + items(&mut reader).await, + [ + "gap", + "0", + "10000", + "gap", + "1000000", + "1010000", + "end 1020000", + "2000000", + "2010000" + ] + ); } #[tokio::test] - async fn continuity_marks_a_latency_skip() { - let mut track = track_producer("continuity", hang::container::track_info(hang::catalog::PRIORITY.audio)); + async fn a_latency_skip_is_a_gap() { + let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); let subscription = track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); @@ -947,8 +1000,63 @@ mod tests { write_group(&mut track, 0, &[ts(0)]); write_group(&mut track, 3, &[ts(1_000_000)]); track.finish().unwrap(); - assert!(!reader.read().await.unwrap().unwrap().continuous); - assert!(!reader.read().await.unwrap().unwrap().continuous); + assert_eq!(items(&mut reader).await, ["gap", "0", "gap", "1000000"]); + } + + /// A video group's trailing duration marker is that frame's end, not a declared stop. + #[tokio::test] + async fn a_trailing_duration_marker_is_not_an_end() { + let track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.video)); + let subscription = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Video)); + let wire = Container::Legacy(crate::container::Kind::Video); + for (sequence, start) in [(0, 0), (1, 33_000)] { + let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let frame = |timestamp: u64, payload| Frame { + timestamp: ts(timestamp), + payload, + keyframe: timestamp == start, + duration: None, + }; + wire.write( + &mut group, + &[ + frame(start, Bytes::from_static(&[1])), + frame(start + 33_000, Bytes::new()), + ], + ) + .unwrap(); + group.finish().unwrap(); + } + track.finish().unwrap(); + assert_eq!(items(&mut reader).await, ["gap", "0", "33000"]); + } + + /// An audio terminal opens its group with the endpoint, then the codec's final packets. + #[tokio::test] + async fn an_audio_terminal_reads_its_end_before_the_final_packets() { + let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscription = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); + write_group(&mut track, 0, &[ts(0), ts(20_000)]); + let mut group = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let frame = |timestamp: u64, payload| Frame { + timestamp: ts(timestamp), + payload, + keyframe: false, + duration: None, + }; + Container::Legacy(crate::container::Kind::Audio) + .write( + &mut group, + &[frame(50_000, Bytes::new()), frame(40_000, Bytes::from_static(&[1]))], + ) + .unwrap(); + group.finish().unwrap(); + track.finish().unwrap(); + assert_eq!(items(&mut reader).await, ["gap", "0", "20000", "end 50000", "40000"]); } #[tokio::test] @@ -963,9 +1071,9 @@ mod tests { write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1); } @@ -985,11 +1093,11 @@ mod tests { write_group(&mut track, 4, &[ts(2_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1, "the hole moves the playhead"); assert_eq!(consumer.markers(), 0, "but the timeline carried on"); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(2_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(2_000_000)); assert_eq!(consumer.discontinuity(), 2); assert_eq!(consumer.markers(), 1, "the marker declares a break"); } @@ -1010,8 +1118,8 @@ mod tests { write_group(&mut track, 2, &[ts(1_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000)); assert_eq!(consumer.discontinuity(), 0, "an empty group is not a marker"); } @@ -1028,9 +1136,9 @@ mod tests { write_group(&mut track, 3, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!( consumer.discontinuity(), 1, @@ -1219,13 +1327,13 @@ mod tests { .write(group, &[frame]) .unwrap(); }; - let read = |consumer: &mut Consumer| match consumer.poll_read(&waiter) { - Poll::Ready(Ok(Some(frame))) => Some(frame.frame.timestamp), - Poll::Pending => None, - other => panic!( - "unexpected read: {:?}", - other.map(|r| r.map(|f| f.map(|f| f.frame.timestamp))) - ), + let read = |consumer: &mut Consumer| loop { + match consumer.poll_read(&waiter) { + Poll::Ready(Ok(Some(Read::Frame { frame, .. }))) => break Some(frame.timestamp), + Poll::Ready(Ok(Some(Read::Gap))) => continue, + Poll::Pending => break None, + other => panic!("unexpected read: {other:?}"), + } }; // A 2.5 s group, half arrived, and the next one already 0.5 s in. @@ -1262,7 +1370,7 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(100_000)); write_group(&mut track, 1, &[ts(0)]); track.finish().unwrap(); let err = consumer.read().await.unwrap_err(); @@ -1296,10 +1404,10 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(100_000)); write_group(&mut track, 1, &[ts(100_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(100_000)); } /// A keyframe one frame below the previous group's last frame, but above its start, is an @@ -1312,7 +1420,7 @@ mod tests { // Read the first group before the second arrives: a blocked group that the next one // already covers is skipped for latency, which is not what this checks. for expected in [0, 33_000, 66_000] { - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(expected)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(expected)); } write_group(&mut track, 1, &[ts(50_000), ts(83_000)]); track.finish().unwrap(); @@ -1349,9 +1457,9 @@ mod tests { write_marker_group(&mut track, 1, ts(0)); write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); assert_eq!(consumer.discontinuity(), 0); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!(consumer.discontinuity(), 1); } @@ -1361,11 +1469,11 @@ mod tests { let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Audio)); consumer.max_delay = Duration::ZERO; write_group(&mut track, 0, &[ts(0)]); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); write_marker_group(&mut track, 1, ts(0)); write_group(&mut track, 2, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(1_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(1_000_000)); assert_eq!( consumer.discontinuity(), 1, @@ -1391,12 +1499,12 @@ mod tests { }], ) .unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); write_marker_group(&mut track, 1, ts(10_000)); write_group(&mut track, 2, &[ts(10_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(10_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(10_000)); assert_eq!( consumer.discontinuity(), 1, @@ -1410,10 +1518,10 @@ mod tests { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(100_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(100_000)); write_group(&mut track, 1, &[ts(200_000), ts(50_000)]); track.finish().unwrap(); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(200_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(200_000)); let err = consumer.read().await.unwrap_err(); assert!(matches!( err, @@ -1499,7 +1607,7 @@ mod tests { group.finish().unwrap(); track.finish().unwrap(); - let frame = consumer.read().await.unwrap().unwrap().frame; + let frame = consumer.read_frame().await.unwrap().unwrap(); assert!(frame.keyframe, "the marker does not consume the first-media slot"); assert!(frame.duration.is_none(), "a leading marker has no previous frame"); } @@ -1686,7 +1794,7 @@ mod tests { let result = tokio::time::timeout(Duration::from_millis(500), async { let mut frames = Vec::new(); while let Ok(Some(frame)) = consumer.read().await { - frames.push(frame.frame); + frames.push(frame); } frames }) @@ -1797,7 +1905,7 @@ mod tests { route.requested_broadcast().await.unwrap().accept(&source); track.demand().used().await.unwrap(); if sequence == 1 { - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(0)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); } for sequence in 1..=sequence { write_group(&mut track, sequence, &[ts(sequence * 1_000_000)]); @@ -1825,7 +1933,7 @@ mod tests { matches!(consumer.poll_event(&waiter), Poll::Ready(Ok(Some(Event::GroupEnd)))), "the complete successor still emits its clean boundary" ); - assert_eq!(consumer.read().await.unwrap().unwrap().frame.timestamp, ts(2_000_000)); + assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(2_000_000)); } // ---- Eviction recovery (pause/resume) ---- @@ -1855,7 +1963,7 @@ mod tests { }], ) .unwrap(); - let first = consumer.read().await.unwrap().unwrap().frame; + let first = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(first.timestamp, ts(0)); // A live group arrives far ahead -- sequences 1..4 never come (evicted). The @@ -1872,12 +1980,9 @@ mod tests { .expect("consumer hung on an evicted group / gap") .unwrap() .unwrap(); - assert_eq!( - next.frame.timestamp, - ts(150_000), - "skipped the evicted gap to the live group" - ); - assert!(!next.continuous, "the eviction skip starts a new span"); + assert!(matches!(next, Read::Gap), "the evicted media is a gap"); + let next = consumer.read_frame().await.unwrap().unwrap(); + assert_eq!(next.timestamp, ts(150_000), "skipped the evicted gap to the live group"); } /// A missing (evicted) sequence with a newer group buffered must be skipped once the @@ -1903,7 +2008,7 @@ mod tests { // Reading must reach group 2 across the gap instead of waiting forever for 1. let reached = tokio::time::timeout(Duration::from_secs(1), async { loop { - let frame = consumer.read().await.unwrap().unwrap().frame; + let frame = consumer.read_frame().await.unwrap().unwrap(); if frame.timestamp == ts(200_000) { return; } @@ -1929,8 +2034,8 @@ mod tests { write_group(&mut track, 0, &[ts(0)]); write_group(&mut track, 2, &[ts(200_000)]); - let first = consumer.read().await.unwrap().expect("group 0 frame"); - assert_eq!(first.frame.timestamp, ts(0)); + let first = consumer.read_frame().await.unwrap().expect("group 0 frame"); + assert_eq!(first.timestamp, ts(0)); // Group 0 is done; group 1's stream is still racing. Within the budget the // consumer waits at the gap rather than skipping to group 2. @@ -2084,7 +2189,7 @@ mod tests { track.finish().unwrap(); // The first frame decodes; the malformed second frame must surface as an error. - let first = consumer.read().await; + let first = consumer.read_frame().await; assert!(matches!(first, Ok(Some(_))), "first frame should decode, got {first:?}"); let second = tokio::time::timeout(Duration::from_millis(200), consumer.read()) @@ -2191,8 +2296,8 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } frames }) @@ -2232,7 +2337,7 @@ mod tests { write_group(&mut track, 0, &[ts(0)]); track.finish().unwrap(); - let frame = consumer.read().await.unwrap().unwrap().frame; + let frame = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(frame.timestamp, ts(0)); consumer.set_max_delay(Duration::from_millis(100)); @@ -2275,8 +2380,8 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } frames }) @@ -2336,8 +2441,8 @@ mod tests { let _frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } frames }) @@ -2482,12 +2587,12 @@ mod tests { // group is served, and nothing else arrives. write_group(&mut track, 100, &[ts(3_000_000)]); - let frame = tokio::time::timeout(Duration::from_millis(200), consumer.read()) + let frame = tokio::time::timeout(Duration::from_millis(200), consumer.read_frame()) .await .expect("an unfloored join must not wait out a gap below the served start") .unwrap() .expect("track still live"); - assert_eq!(frame.frame.timestamp, ts(3_000_000)); + assert_eq!(frame.timestamp, ts(3_000_000)); } #[tokio::test] @@ -2666,7 +2771,7 @@ mod tests { track.finish().unwrap(); // Take the first frame only, leaving the second unread. - let first = consumer.read().await.unwrap().unwrap().frame; + let first = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(first.timestamp, ts(0)); // Expiry aborts the finished group while the reader is still inside it. @@ -2741,8 +2846,8 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } frames }) @@ -2798,8 +2903,8 @@ mod tests { track.finish().unwrap(); let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } assert_eq!(frames.len(), 3); @@ -2840,8 +2945,8 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } frames }) @@ -2876,8 +2981,8 @@ mod tests { group1.finish().unwrap(); track.finish().unwrap(); let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } assert_eq!(frames.len(), 2); assert_eq!(consumer.discontinuity(), 0); @@ -2915,8 +3020,8 @@ mod tests { let frames = tokio::time::timeout(Duration::from_secs(2), async { let mut frames = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - frames.push(frame.frame); + while let Some(frame) = consumer.read_frame().await.unwrap() { + frames.push(frame); } frames }) @@ -2946,14 +3051,14 @@ mod tests { }], ) .unwrap(); - let frame = tokio::time::timeout(Duration::from_secs(1), consumer.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), consumer.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.frame.timestamp, ts(0)); + assert_eq!(frame.timestamp, ts(0)); write_marker(&mut group, ts(15_000)); let event = kio::wait(|waiter| consumer.poll_event(waiter)).await.unwrap(); - assert!(matches!(event, Some(Event::FrameEnd(end)) if end == ts(15_000))); + assert!(matches!(event, Some(Event::FrameEnd { end, opens: false }) if end == ts(15_000))); } } diff --git a/rs/moq-mux/src/container/flv/import_test.rs b/rs/moq-mux/src/container/flv/import_test.rs index 618118ba95f..3f0b3b29349 100644 --- a/rs/moq-mux/src/container/flv/import_test.rs +++ b/rs/moq-mux/src/container/flv/import_test.rs @@ -181,13 +181,10 @@ async fn import_emits_frames() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = decoder.read().await.unwrap().expect("a video frame"); - assert!(frame.frame.keyframe); + let frame = decoder.read_frame().await.unwrap().expect("a video frame"); + assert!(frame.keyframe); // The payload is the length-prefixed NALU, carried through verbatim. - assert_eq!( - frame.frame.payload.as_ref(), - &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00] - ); + assert_eq!(frame.payload.as_ref(), &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00]); } #[tokio::test(start_paused = true)] @@ -210,15 +207,12 @@ async fn public_container_preserves_loc_for_flv() { track, crate::catalog::hang::Container::Loc(crate::container::Kind::Data), ); - let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!( - frame.frame.payload.as_ref(), - &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00] - ); + assert_eq!(frame.payload.as_ref(), &[0, 0, 0, 5, 0x65, 0x88, 0x84, 0x21, 0x00]); } /// Bytes split across two `decode` calls still reassemble into whole tags. @@ -510,9 +504,9 @@ async fn import_enhanced_av1() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = decoder.read().await.unwrap().expect("an AV1 frame"); - assert!(frame.frame.keyframe); - assert_eq!(frame.frame.payload.as_ref(), payload); + let frame = decoder.read_frame().await.unwrap().expect("an AV1 frame"); + assert!(frame.keyframe); + assert_eq!(frame.payload.as_ref(), payload); } /// GStreamer 1.28 `fdkaacenc` output at 48 kHz stereo, remuxed to FLV by ffmpeg 9.0.1, whose @@ -644,8 +638,8 @@ async fn import_reports_negative_pts_and_can_resume() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = decoder.read().await.unwrap().expect("the good frame"); - assert_eq!(frame.frame.timestamp.as_millis(), 10); + let frame = decoder.read_frame().await.unwrap().expect("the good frame"); + assert_eq!(frame.timestamp.as_millis(), 10); } #[tokio::test(start_paused = true)] @@ -684,8 +678,8 @@ async fn import_enhanced_hvc1_applies_composition_time() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = decoder.read().await.unwrap().expect("a video frame"); - assert_eq!(frame.frame.timestamp.as_millis(), 17); + let frame = decoder.read_frame().await.unwrap().expect("a video frame"); + assert_eq!(frame.timestamp.as_millis(), 17); } #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/container/fmp4/export.rs b/rs/moq-mux/src/container/fmp4/export.rs index 81094b85080..686107e362c 100644 --- a/rs/moq-mux/src/container/fmp4/export.rs +++ b/rs/moq-mux/src/container/fmp4/export.rs @@ -563,7 +563,7 @@ impl Export { track.pending = Some(frame); break; } - Poll::Ready(Some(Event::FrameEnd(end))) => { + Poll::Ready(Some(Event::FrameEnd { end, .. })) => { if track.is_video && let Some(last) = track.buffer.last_mut() && last.duration.is_none() diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index da0a0111908..ea2b2f7c503 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -1815,7 +1815,7 @@ async fn webvtt_samples_become_timed_text_with_source_metadata() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap().frame; + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!(frame.timestamp.as_micros(), 250_000); assert_eq!( &frame.payload[..], @@ -1885,7 +1885,7 @@ async fn webvtt_kind_and_lone_cr_terminators() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap().frame; + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!( &frame.payload[..], b"WEBVTT\nKind: captions\n\n00:00:00.250 --> 00:00:01.500\nHello\nworld\n\n" @@ -1996,6 +1996,6 @@ async fn webvtt_sub_millisecond_cue_stays_positive() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap().frame; + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!(&frame.payload[..], b"WEBVTT\n\n00:00:00.250 --> 00:00:00.251\nhi\n\n"); } diff --git a/rs/moq-mux/src/container/group.rs b/rs/moq-mux/src/container/group.rs index 8ac2d9052b2..31fcd7d84e4 100644 --- a/rs/moq-mux/src/container/group.rs +++ b/rs/moq-mux/src/container/group.rs @@ -144,11 +144,11 @@ mod tests { subscription, Hang::new(&config, crate::container::Kind::Data).unwrap(), ); - let first = live.read().await.unwrap().unwrap(); - assert!(first.frame.payload.is_empty()); - assert!(first.frame.keyframe); - assert_eq!(live.read().await.unwrap().unwrap().frame.payload, b"data".as_slice()); - assert!(live.read().await.unwrap().is_none()); + let first = live.read_frame().await.unwrap().unwrap(); + assert!(first.payload.is_empty()); + assert!(first.keyframe); + assert_eq!(live.read_frame().await.unwrap().unwrap().payload, b"data".as_slice()); + assert!(live.read_frame().await.unwrap().is_none()); let group = consumer.track("data").unwrap().fetch_group(0, None).await.unwrap(); let mut fetched = GroupConsumer::new(group, Hang::new(&config, crate::container::Kind::Data).unwrap()); assert!(fetched.read().await.unwrap().unwrap().payload.is_empty()); diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index 0310016ee3a..9a1e8f080bb 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -263,12 +263,12 @@ async fn public_container_preserves_loc_for_mkv() { track, crate::catalog::hang::Container::Loc(crate::container::Kind::Data), ); - let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.frame.payload.as_ref(), payload); + assert_eq!(frame.payload.as_ref(), payload); } #[test] @@ -683,7 +683,7 @@ async fn subtitle_languages_and_empty_tracks_are_published() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap().frame; + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!( &frame.payload[..], b"WEBVTT\n\n00:00:00.250 --> 00:00:01.500\nHello\n\n" @@ -730,7 +730,7 @@ async fn subtitle_duration_survives_as_a_timed_cue() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap().frame; + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!(frame.timestamp.as_micros(), 250_000); assert_eq!( &frame.payload[..], @@ -786,7 +786,7 @@ async fn webvtt_preserves_cue_metadata_and_relative_timestamps() { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let frame = reader.read().await.unwrap().unwrap().frame; + let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!(&frame.payload[..], b"WEBVTT\n\nSTYLE\n::cue { color: lime; }\n\nNOTE source\n\ncue-1\n00:00:00.250 --> 00:00:01.500 align:start\nHello <00:00:00.750>world\n\n"); } @@ -904,11 +904,11 @@ async fn laced_opus_frames() { let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::try_from(config).unwrap()); for (index, packet) in packets.iter().enumerate() { - let frame = reader.read().await.unwrap().expect("one frame per packet").frame; + let frame = reader.read_frame().await.unwrap().expect("one frame per packet"); assert_eq!(frame.payload.as_ref(), *packet, "{lacing:?}, grouped={grouped}"); assert_eq!(frame.timestamp.as_micros(), 107_000 + index as u128 * 20_000); } - assert!(reader.read().await.unwrap().is_none()); + assert!(reader.read_frame().await.unwrap().is_none()); } } } diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 77670d47115..c3ebc30641b 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -45,13 +45,28 @@ pub enum Kind { Data, } -/// A delivered frame and whether it continues the previous delivery. +/// One item of a [`Consumer`] read, in delivery order. #[derive(Clone, Debug)] -pub struct Delivery { - /// The encoded or decoded media frame. - pub frame: T, - /// False on the first frame and after a playhead discontinuity. - pub continuous: bool, +pub enum Read { + /// A media frame from `group`. + Frame { + /// The group's sequence number. Only equality means anything: numbers may skip. + group: u64, + frame: Frame, + }, + /// Media before the next item will never reach this reader: it joined mid-track, or a + /// group was skipped, evicted, or never arrived. A decoder resumes from here without + /// the references or state that media built. + Gap, + /// The publisher declared that its media stops at `end`: it paused, switched source, or + /// finished. Frames that follow from the same `group` are the stopped encoder's final + /// packets (an audio codec's drain), whose output at or past `end` is padding. The next + /// group starts fresh. + End { + /// The group's sequence number. Only equality means anything: numbers may skip. + group: u64, + end: moq_net::Timestamp, + }, } /// A decoded media frame: timestamp, payload bytes, keyframe flag. diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index 02068a33af8..638215537ab 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -1687,8 +1687,8 @@ mod tests { let mut consumer = crate::container::Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Video)); let mut timestamps = Vec::new(); - while let Some(frame) = consumer.read().await.unwrap() { - timestamps.push(frame.frame.timestamp.as_micros()); + while let Some(frame) = consumer.read_frame().await.unwrap() { + timestamps.push(frame.timestamp.as_micros()); } assert_eq!(timestamps, [1_000_000, 1_040_000, 1_050_000]); } diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index 9c25239af38..2fcc76aad27 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -315,7 +315,7 @@ impl ExportSource { loop { match ready!(self.poll_event(waiter))? { Some(Event::Frame(frame)) => return Poll::Ready(Ok(Some(frame))), - Some(Event::GroupEnd | Event::FrameEnd(_)) => continue, + Some(Event::GroupEnd | Event::FrameEnd { .. }) => continue, None => return Poll::Ready(Ok(None)), } } diff --git a/rs/moq-mux/src/container/test_util.rs b/rs/moq-mux/src/container/test_util.rs index 05c7a80c9e3..9c65a3995eb 100644 --- a/rs/moq-mux/src/container/test_util.rs +++ b/rs/moq-mux/src/container/test_util.rs @@ -152,12 +152,12 @@ pub(crate) async fn published( let track = consumer.track(&name).unwrap().subscribe(replay).await.unwrap(); let mut reader = crate::container::Consumer::new(track, container); let mut timestamps = Vec::new(); - while let Some(frame) = tokio::time::timeout(std::time::Duration::from_secs(5), reader.read()) + while let Some(frame) = tokio::time::timeout(std::time::Duration::from_secs(5), reader.read_frame()) .await .expect("the importer finished its tracks") .unwrap() { - timestamps.push(frame.frame.timestamp.as_micros()); + timestamps.push(frame.timestamp.as_micros()); } out.insert(name, timestamps); } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index ca005988f6e..b0a68d6dff3 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -1305,12 +1305,12 @@ async fn export_scte35_roundtrip() { .unwrap(); let mut scte_reader = crate::container::Consumer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); let frame = scte_reader - .read() + .read_frame() .await .unwrap() .expect("no SCTE-35 frame after round-trip"); assert_eq!( - frame.frame.payload.as_ref(), + frame.payload.as_ref(), CUE, "SCTE-35 section did not round-trip byte-for-byte" ); @@ -1424,12 +1424,12 @@ async fn export_pes_verbatim_roundtrip() { .unwrap(); let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); let frame = reader - .read() + .read_frame() .await .unwrap() .expect("no verbatim PES frame after round-trip"); assert_eq!( - frame.frame.payload.as_ref(), + frame.payload.as_ref(), PAYLOAD, "verbatim PES payload round-trips byte-for-byte" ); @@ -1502,9 +1502,9 @@ async fn read_frames(consumer: &moq_net::broadcast::Consumer, name: &str, kind: .unwrap(); let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy(kind)); let mut frames = Vec::new(); - while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { + while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { let Some(frame) = res.unwrap() else { break }; - frames.push(frame.frame.payload.to_vec()); + frames.push(frame.payload.to_vec()); } frames } @@ -2083,9 +2083,9 @@ async fn read_cues(consumer: &moq_net::broadcast::Consumer, name: &str) -> Vec<( .unwrap(); let mut reader = crate::container::Consumer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); let mut cues = Vec::new(); - while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { + while let Ok(res) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { let Some(frame) = res.unwrap() else { break }; - cues.push((frame.frame.payload.to_vec(), frame.frame.timestamp)); + cues.push((frame.payload.to_vec(), frame.timestamp)); } cues } diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index bf4ad33e814..36a4cd7fc5f 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -3885,13 +3885,13 @@ pub(super) mod test { .await .unwrap(); let mut reader = Consumer::new(track, Container::Legacy(crate::container::Kind::Data)); - let frame = tokio::time::timeout(Duration::from_secs(1), reader.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), reader.read_frame()) .await .expect("cue read timed out") .unwrap() .expect("a published cue frame"); assert_eq!( - &frame.frame.payload[..], + &frame.payload[..], &CUE[..], "verbatim splice_info_section after upgrade" ); @@ -4554,8 +4554,8 @@ pub(super) mod test { crate::catalog::hang::Container::Legacy(crate::container::Kind::Audio), ); let mut frames = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { - frames.push(frame.frame); + while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { + frames.push(frame); } frames } @@ -4608,8 +4608,8 @@ pub(super) mod test { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let mut frames = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { - frames.push(frame.frame); + while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { + frames.push(frame); } frames } @@ -5432,7 +5432,7 @@ pub(super) mod test { track, crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); - let published = tokio::time::timeout(Duration::from_millis(50), reader.read()).await; + let published = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await; assert!( matches!(published, Ok(Ok(Some(_)))), "a PID that never published held the catalog shut: {published:?}" @@ -5822,18 +5822,18 @@ pub(super) mod test { .await .unwrap(); let mut reader = Consumer::new(track, Container::Legacy(crate::container::Kind::Data)); - let frame = tokio::time::timeout(Duration::from_secs(1), reader.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), reader.read_frame()) .await .expect("cue read timed out") .unwrap() .expect("a published cue frame"); - assert_eq!(&frame.frame.payload[..], &CUE[..], "verbatim splice_info_section"); - assert_ne!(frame.frame.timestamp, Timestamp::ZERO, "cue must not stamp zero"); + assert_eq!(&frame.payload[..], &CUE[..], "verbatim splice_info_section"); + assert_ne!(frame.timestamp, Timestamp::ZERO, "cue must not stamp zero"); // The legacy container normalizes the wire timestamp to microseconds, so compare the // instant (not the raw scale) against the 90 kHz media clock the cue was stamped with. assert_eq!( - Duration::from(frame.frame.timestamp), + Duration::from(frame.timestamp), Duration::from(clock), "cue stamped with the video media clock" ); @@ -6008,16 +6008,12 @@ pub(super) mod test { .await .unwrap(); let mut reader = Consumer::new(track, Container::Legacy(crate::container::Kind::Data)); - let frame = tokio::time::timeout(Duration::from_secs(1), reader.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), reader.read_frame()) .await .expect("verbatim read timed out") .unwrap() .expect("a published verbatim frame"); - assert_eq!( - &frame.frame.payload[..], - &payload[..], - "verbatim PES payload round-trips" - ); + assert_eq!(&frame.payload[..], &payload[..], "verbatim PES payload round-trips"); } /// A verbatim-only program retains the PES clock while waiting for rate measurement or EOF. @@ -6103,9 +6099,10 @@ pub(super) mod test { crate::catalog::hang::Container::Legacy(crate::container::Kind::Audio), ); let mut frames = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await + while let Ok(Ok(Some(frame))) = + tokio::time::timeout(std::time::Duration::from_millis(50), reader.read_frame()).await { - frames.push(frame.frame); + frames.push(frame); } (frames, reader.discontinuity()) } @@ -6711,9 +6708,9 @@ pub(super) mod test { let track = consumer.track(name).unwrap().subscribe(subscription).await.unwrap(); let mut reader = crate::container::Consumer::new(track, crate::catalog::hang::Container::Legacy(kind)); let mut frames = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read()).await { - if !frame.frame.payload.is_empty() { - frames.push(frame.frame); + while let Ok(Ok(Some(frame))) = tokio::time::timeout(Duration::from_millis(50), reader.read_frame()).await { + if !frame.payload.is_empty() { + frames.push(frame); } } frames diff --git a/rs/moq-mux/src/container/ts/import_test.rs b/rs/moq-mux/src/container/ts/import_test.rs index 863033a02b7..5a31923c29b 100644 --- a/rs/moq-mux/src/container/ts/import_test.rs +++ b/rs/moq-mux/src/container/ts/import_test.rs @@ -96,13 +96,13 @@ async fn public_container_preserves_loc_for_ts() { track, crate::catalog::hang::Container::Loc(crate::container::Kind::Data), ); - let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(std::time::Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert!(frame.frame.keyframe); - assert!(!frame.frame.payload.is_empty()); + assert!(frame.keyframe); + assert!(!frame.payload.is_empty()); } /// The Kyrion capture is H.264 1080i with B-frames (open-GOP, ~5-frame reorder despite only @@ -262,9 +262,9 @@ async fn import_opus_frames() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let mut frames = Vec::new(); - while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { + while let Ok(res) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read_frame()).await { let Some(frame) = res.unwrap() else { break }; - frames.push(frame.frame.payload); + frames.push(frame.payload); } assert!(frames.len() > 5, "expected a run of Opus packets, got {}", frames.len()); @@ -541,14 +541,13 @@ async fn survives_midstream_join() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let mut frames = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { + while let Ok(Ok(Some(frame))) = + tokio::time::timeout(std::time::Duration::from_millis(50), reader.read_frame()).await + { frames.push(frame); } assert_eq!(frames.len(), 1, "expected only the post-join IDR, got {}", frames.len()); - assert!( - frames[0].frame.keyframe, - "the first surviving frame must be the keyframe" - ); + assert!(frames[0].keyframe, "the first surviving frame must be the keyframe"); } /// A real Ateme Kyrion broadcast captured mid-stream with `nc`, so it opens dirty: @@ -598,8 +597,10 @@ async fn kyrion_dirtystart_extracts_real_cues() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let mut cues = Vec::new(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(std::time::Duration::from_millis(50), reader.read()).await { - cues.push((frame.frame.payload.to_vec(), frame.frame.timestamp)); + while let Ok(Ok(Some(frame))) = + tokio::time::timeout(std::time::Duration::from_millis(50), reader.read_frame()).await + { + cues.push((frame.payload.to_vec(), frame.timestamp)); } assert_eq!(cues.len(), 6, "expected the six real splice_inserts"); assert!( diff --git a/rs/moq-mux/src/container/ts/programs.rs b/rs/moq-mux/src/container/ts/programs.rs index 10e13f53cb5..5324bd19e8f 100644 --- a/rs/moq-mux/src/container/ts/programs.rs +++ b/rs/moq-mux/src/container/ts/programs.rs @@ -376,16 +376,17 @@ mod test { let (name, config) = renditions[0]; let track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let container = Container::try_from(config).unwrap(); - let frame = Consumer::new(track, container).read().await.unwrap().expect("a frame"); - assert!( - fills.contains(&frame.frame.payload[4]), - "{path} carries only its own program" - ); + let frame = Consumer::new(track, container) + .read_frame() + .await + .unwrap() + .expect("a frame"); + assert!(fills.contains(&frame.payload[4]), "{path} carries only its own program"); let skew = Duration::from_secs(2); let wall = catalog .clock .expect("the catalog advertises a clock") - .wall_clock(frame.frame.timestamp) + .wall_clock(frame.timestamp) .unwrap(); assert!( before - skew <= wall && wall <= after + skew, diff --git a/rs/moq-mux/src/import/track.rs b/rs/moq-mux/src/import/track.rs index 208b58d8fed..58582eb5059 100644 --- a/rs/moq-mux/src/import/track.rs +++ b/rs/moq-mux/src/import/track.rs @@ -1050,13 +1050,13 @@ mod tests { .decode(&payload, Some(Timestamp::from_micros(1_000).unwrap())) .unwrap(); - let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.frame.payload, payload); - assert_eq!(frame.frame.timestamp, Timestamp::from_micros(1_000).unwrap()); + assert_eq!(frame.payload, payload); + assert_eq!(frame.timestamp, Timestamp::from_micros(1_000).unwrap()); import.finish().unwrap(); } @@ -1378,14 +1378,14 @@ mod tests { .decode(&au, Some(Timestamp::from_micros(1_000).unwrap())) .unwrap(); - let frame = tokio::time::timeout(Duration::from_secs(1), media.read()) + let frame = tokio::time::timeout(Duration::from_secs(1), media.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert!(frame.frame.keyframe, "an IDR access unit must open a group"); - assert_eq!(frame.frame.payload, au, "the payload must pass through verbatim"); - assert_eq!(frame.frame.timestamp, Timestamp::from_micros(1_000).unwrap()); + assert!(frame.keyframe, "an IDR access unit must open a group"); + assert_eq!(frame.payload, au, "the payload must pass through verbatim"); + assert_eq!(frame.timestamp, Timestamp::from_micros(1_000).unwrap()); import.finish().unwrap(); } diff --git a/rs/moq-rtc/src/codec/mod.rs b/rs/moq-rtc/src/codec/mod.rs index 5b3a6c4f0ff..20d24b7a712 100644 --- a/rs/moq-rtc/src/codec/mod.rs +++ b/rs/moq-rtc/src/codec/mod.rs @@ -230,17 +230,20 @@ impl Track { /// Pull the next RTP-ready frame. Returns `None` when the track ends. pub async fn next(&mut self) -> Result> { loop { - let Some(frame) = self.consumer.read().await? else { - return Ok(None); + let frame = match self.consumer.read().await? { + Some(moq_mux::container::Read::Frame { frame, .. }) => frame, + // RTP sequence numbers and timestamps already show the hole to a receiver. + Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. }) => continue, + None => return Ok(None), }; let payload = match &self.convert { - TrackConvert::Passthrough => frame.frame.payload, + TrackConvert::Passthrough => frame.payload, TrackConvert::LengthPrefixed { length_size, keyframe_prefix, } => { - let prefix = frame.frame.keyframe.then(|| keyframe_prefix.as_ref()); - moq_mux::codec::annexb::from_length_prefixed(&frame.frame.payload, *length_size, prefix) + let prefix = frame.keyframe.then(|| keyframe_prefix.as_ref()); + moq_mux::codec::annexb::from_length_prefixed(&frame.payload, *length_size, prefix) .map_err(moq_mux::Error::from)? } }; @@ -248,7 +251,7 @@ impl Track { continue; } return Ok(Some(PacketizedFrame { - timestamp_us: frame.frame.timestamp.as_micros() as u64, + timestamp_us: frame.timestamp.as_micros() as u64, payload, })); } diff --git a/rs/moq-srt/src/ts.rs b/rs/moq-srt/src/ts.rs index c5a01ef9d09..8af41f002de 100644 --- a/rs/moq-srt/src/ts.rs +++ b/rs/moq-srt/src/ts.rs @@ -647,12 +647,11 @@ mod tests { let track = broadcast.track(name.as_str()).unwrap().subscribe(None).await.unwrap(); let mut reader = moq_mux::container::Consumer::new(track, Container::Legacy(moq_mux::container::Kind::Data)); - let cue = timeout(Duration::from_secs(5), reader.read()) + let cue = timeout(Duration::from_secs(5), next_frame(&mut reader)) .await .expect("cue read timed out") - .unwrap() .expect("a published cue section"); - let expected_cue = cue.frame.payload; + let expected_cue = cue.payload; assert_eq!(expected_cue[0], 0xFC, "a verbatim splice_info_section (table_id 0xFC)"); let mut subscriber = Subscriber::new(&origin.consume(), "ingest", Options::default()) @@ -696,18 +695,24 @@ mod tests { .expect("SRT egress preserves the SCTE-35 track"); let track = roundtrip_consumer.track(name).unwrap().subscribe(None).await.unwrap(); let mut reader = moq_mux::container::Consumer::new(track, Container::Legacy(moq_mux::container::Kind::Data)); - let cue = timeout(Duration::from_secs(5), reader.read()) + let cue = timeout(Duration::from_secs(5), next_frame(&mut reader)) .await .expect("round-trip cue read timed out") - .unwrap() .expect("SRT egress preserves a cue section"); + assert_eq!(cue.payload[0], 0xFC, "the round-trip cue is a splice_info_section"); assert_eq!( - cue.frame.payload[0], 0xFC, - "the round-trip cue is a splice_info_section" - ); - assert_eq!( - cue.frame.payload, expected_cue, + cue.payload, expected_cue, "SRT egress preserves the complete cue section" ); } + + /// The next frame, passing over gaps and ends. + async fn next_frame(reader: &mut moq_mux::container::Consumer) -> Option { + loop { + match reader.read().await.unwrap()? { + moq_mux::container::Read::Frame { frame, .. } => return Some(frame), + moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. } => continue, + } + } + } } diff --git a/rs/moq-video/src/capture/wgc.rs b/rs/moq-video/src/capture/wgc.rs index d426fc0b6fb..f2d44221d95 100644 --- a/rs/moq-video/src/capture/wgc.rs +++ b/rs/moq-video/src/capture/wgc.rs @@ -327,8 +327,12 @@ mod tests { let (surface, timestamp) = delivery.next(now).unwrap().unwrap(); let frame = crate::Frame::new(crate::Surface::I420(surface.clone()), timestamp); producer.publish(&encoder.encode(&frame).unwrap()).unwrap(); - let received = subscriber.read().await.unwrap().unwrap(); - assert_eq!(received.frame.timestamp.as_micros(), u128::from(step) * 20_000); + let received = loop { + if let moq_mux::container::Read::Frame { frame, .. } = subscriber.read().await.unwrap().unwrap() { + break frame; + } + }; + assert_eq!(received.timestamp.as_micros(), u128::from(step) * 20_000); } } diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 4ecf60d516a..327e3988017 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -3,6 +3,7 @@ use std::collections::VecDeque; use hang::catalog::VideoConfig; +use moq_mux::container::Read; use super::decoder::Config; use super::sink::Sink; @@ -85,8 +86,6 @@ pub struct Consumer { pending: VecDeque, /// Whether the ended track's decoder has already been drained. drained: bool, - /// Last container playhead generation observed. - discontinuity: u64, } impl Consumer { @@ -140,7 +139,6 @@ impl Consumer { track, pending: VecDeque::new(), drained: false, - discontinuity: 0, }) } @@ -166,14 +164,13 @@ impl Consumer { return Ok(None); } - let mux_frame = self.track.read().await?; - let discontinuity = self.track.discontinuity(); - if discontinuity != self.discontinuity { - // A playhead event re-applies startup delay and skip; the next group - // already starts on a keyframe with parameter sets, so the decoder is - // not flushed. - self.discontinuity = discontinuity; - } + let mux_frame = match self.track.read().await? { + Some(Read::Frame { frame, .. }) => Some(frame), + // The next group starts on a keyframe with parameter sets, so neither a + // gap nor a declared end flushes the decoder. + Some(Read::Gap | Read::End { .. }) => continue, + None => None, + }; let Some(mux_frame) = mux_frame else { // The flag goes up only once the tail is in hand, so a read @@ -194,11 +191,7 @@ impl Consumer { self.pending.extend( self.decoder - .decode( - mux_frame.frame.payload, - mux_frame.frame.timestamp, - mux_frame.frame.keyframe, - ) + .decode(mux_frame.payload, mux_frame.timestamp, mux_frame.keyframe) .await?, ); } diff --git a/rs/moq-video/src/decode/decoder.rs b/rs/moq-video/src/decode/decoder.rs index 7bfd320c410..5ce8d34910e 100644 --- a/rs/moq-video/src/decode/decoder.rs +++ b/rs/moq-video/src/decode/decoder.rs @@ -598,14 +598,19 @@ mod tests { .unwrap(); let mut track = moq_mux::container::Consumer::new(track, moq_mux::catalog::hang::Container::try_from(config).unwrap()); - let sample = track.read().await.unwrap().expect("an imported sample"); - assert!(sample.frame.keyframe); + let sample = loop { + let read = track.read().await.unwrap().expect("an imported sample"); + if let moq_mux::container::Read::Frame { frame, .. } = read { + break frame; + } + }; + assert!(sample.keyframe); let decoder = super::Decoder::new(config, &decode_config(super::Kind::Named(probe::BUFFERED_NAME.into()))).unwrap(); let access_unit = decoder .conversion - .access_unit(&sample.frame.payload, sample.frame.keyframe) + .access_unit(&sample.payload, sample.keyframe) .unwrap(); let mut buf = access_unit.clone(); let mut nals = moq_mux::codec::annexb::NalIterator::new(&mut buf); diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index de8a7aa3227..3241d3c096a 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -1208,8 +1208,12 @@ mod tests { } async fn read(track: &mut moq_mux::container::Consumer) -> u64 { - let frame = track.read().await.unwrap().expect("a published frame"); - u64::try_from(frame.frame.timestamp.as_micros()).unwrap() + loop { + let read = track.read().await.unwrap().expect("a published frame"); + if let moq_mux::container::Read::Frame { frame, .. } = read { + return u64::try_from(frame.timestamp.as_micros()).unwrap(); + } + } } /// Read the next frame not already in `seen`: a resubscription replays retained groups first. From c5817e3cfff5240ea418338bd2797911cf700bdc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 19:59:38 -0700 Subject: [PATCH 14/19] fix(interop): read the frame item in the JS native subscriber Co-Authored-By: Claude Opus 5.5 --- test/interop/clients/js-native/subscribe.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/interop/clients/js-native/subscribe.ts b/test/interop/clients/js-native/subscribe.ts index 4a11e8f3677..b7b3b17f982 100644 --- a/test/interop/clients/js-native/subscribe.ts +++ b/test/interop/clients/js-native/subscribe.ts @@ -129,7 +129,7 @@ async function run(): Promise { for (;;) { const next = await consumer.next(); if (!next) break; - const bytes = next.frame?.payload.byteLength ?? 0; + const bytes = next.kind === "frame" ? next.frame.payload.byteLength : 0; if (bytes > 0) { // The harness judges success by this marker, not the exit code: the // @moq/web-transport NAPI addon can segfault during the runtime's exit From be9337e8f92bf1a7e91d36624cdaf4215a026621 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 20:17:42 -0700 Subject: [PATCH 15/19] fix(hang): read no gap when a retired group lost nothing Retiring a fully delivered group, or one holding only a trailing endpoint, for the max delay read as a gap. Only unread media, a shed marker group, or a broken timeline is a gap, matching the Rust skip path. Co-Authored-By: Claude Opus 5.5 --- js/hang/src/container/consumer.test.ts | 47 ++++++++++++++++++++++++++ js/hang/src/container/consumer.ts | 13 +++---- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index a2c740a4002..70aa40ec9bc 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -965,6 +965,53 @@ test("Consumer accepts open-GOP leading pictures above the previous group", asyn consumer.close(); }); +// A delivered group still awaiting its done result holds no unread media, so retiring it for +// the max delay loses nothing. +test("Consumer reads no gap when retiring a fully delivered group", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 150 as Time.Milli }); + + writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); + await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + + writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); + await settle(); + + // Read directly: a gap here would fail the frame check. + expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); + + consumer.close(); +}); + +// An unread trailing endpoint carries no media, so retiring its group loses nothing. +test("Consumer reads no gap when retiring a group holding only its endpoint", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("video"), maxDelay: 150 as Time.Milli }); + + const group = new Group.Producer(0); + group.writeFrame({ payload: encodeLegacy(0 as Time.Micro), timestamp: Time.Timestamp.now() }); + group.writeFrame({ + payload: encodeLegacyFrame(33_000 as Time.Micro, new Uint8Array()), + timestamp: Time.Timestamp.now(), + }); + group.close(); + track.writeGroup(group); + await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + + writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); + await settle(); + + expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); + + consumer.close(); +}); + // --- Buffered signal --- test("Consumer buffered signal updates as frames arrive", async () => { diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index a0670f027e3..1351bdfe1c0 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -360,13 +360,14 @@ export class Consumer { this.#groups.shift(); this.#active = this.#groups[0]?.consumer.sequence; - const nextStart = this.#groups[0]?.frames.at(0)?.timestamp ?? this.#groups[0]?.end; + const next = this.#groups[0]; + const nextStart = next?.frames.at(0)?.timestamp ?? next?.end; const marker = !first.empty && !first.media; - // Buffered frames are thrown away, and `first.end` counts them, so contiguity alone would hide them. - const discarded = first.frames.length > 0; - if (marker || discarded || !ptsContiguous(first.end ?? this.#presentedEnd, nextStart)) { - this.#lost = true; - } + // Unread media is thrown away, and `first.end` counts it, so contiguity alone would hide it. + // A trailing endpoint carries no media, so leaving it unread discards nothing. + const discarded = first.frames.some((frame) => this.#format.end?.(frame) === undefined); + const continued = continues(first, next) || ptsContiguous(first.end ?? this.#presentedEnd, nextStart); + if (marker || discarded || !continued) this.#lost = true; first.consumer.close(); first.frames.length = 0; skipped++; From b5cef36f17157356f06a42c63b45fa08582a5be9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 20:41:53 -0700 Subject: [PATCH 16/19] test(watch): narrow arrival entries to frames Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/replay.test.ts | 2 +- js/watch/src/video/decoder.test.ts | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts index 24e07956a51..d39ba7f4534 100644 --- a/js/watch/src/audio/replay.test.ts +++ b/js/watch/src/audio/replay.test.ts @@ -157,7 +157,7 @@ describe.each(["shared", "post"] as const)("%s ingress diagnostics", (ring) => { expect(held).toEqual([ { kind: "gap" }, ...trace.map((entry) => ({ - kind: "frame", + kind: "frame" as const, timestamp: Time.Milli(entry.timestamp), arrival: Time.Milli(entry.at), late: false, diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 8feb7e5904e..35e0f8efea2 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -441,7 +441,10 @@ for (const kind of ["legacy", "cmaf"] as const) { try { expect(playback.submitted).toEqual([1000, 1100, 1200]); for (const timestamp of [1, 1.1, 1.2]) - expect((await playback.arrivals.next())?.timestamp).toBe(Time.Milli(timestamp)); + expect(await playback.arrivals.next()).toMatchObject({ + kind: "frame", + timestamp: Time.Milli(timestamp), + }); // Rejected payloads were still received, but never counted as frames. expect(playback.decoder.out.stats.peek()).toEqual({ frameCount: 3, bytesReceived: 5 }); } finally { From 552b23bcb89a009327cae0f476c9cb54d81c388c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 12 Oct 2026 22:05:34 -0700 Subject: [PATCH 17/19] fix(mux): read a joined marker's end, and keep gaps for unfinished retired groups Rust startup now adopts the first group carrying any wire frame, so joining a paused track whose only served group is a marker reads the gap and end instead of waiting for media. JS retirement for the max delay only treats an adjacent successor as continuous once the retired group finished cleanly; a truncated or still-open group may have lost its tail. Co-Authored-By: Claude Opus 5.5 --- js/hang/src/container/consumer.test.ts | 47 ++++++++++++++++++++++++++ js/hang/src/container/consumer.ts | 8 +++-- rs/moq-mux/src/container/consumer.rs | 31 ++++++++++++++--- 3 files changed, 79 insertions(+), 7 deletions(-) diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 70aa40ec9bc..8d9a9f8f4c1 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -1012,6 +1012,53 @@ test("Consumer reads no gap when retiring a group holding only its endpoint", as consumer.close(); }); +// A truncated group lost its tail, so retiring it for the max delay is a gap even when the +// next group is adjacent. +test("Consumer reads a gap when retiring a truncated group", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 150 as Time.Milli }); + + const group = new Group.Producer(0); + group.writeFrame({ payload: encodeLegacy(0 as Time.Micro), timestamp: Time.Timestamp.now() }); + track.writeGroup(group); + await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + + group.close(new NetError.Stream(StreamCode.Cancel)); + writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); + await settle(); + + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); + + consumer.close(); +}); + +// An open group may still have a tail in flight, so adjacency proves nothing when it is retired. +test("Consumer reads a gap when retiring an open group", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 150 as Time.Milli }); + + const group = new Group.Producer(0); + group.writeFrame({ payload: encodeLegacy(0 as Time.Micro), timestamp: Time.Timestamp.now() }); + track.writeGroup(group); + await settle(); + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + + writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); + writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); + await settle(); + + expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); + + group.close(); + consumer.close(); +}); + // --- Buffered signal --- test("Consumer buffered signal updates as frames arrive", async () => { diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 1351bdfe1c0..a4bf7c3c188 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -366,8 +366,12 @@ export class Consumer { // Unread media is thrown away, and `first.end` counts it, so contiguity alone would hide it. // A trailing endpoint carries no media, so leaving it unread discards nothing. const discarded = first.frames.some((frame) => this.#format.end?.(frame) === undefined); - const continued = continues(first, next) || ptsContiguous(first.end ?? this.#presentedEnd, nextStart); - if (marker || discarded || !continued) this.#lost = true; + // Adjacency proves nothing is missing only once the group finished cleanly: a truncated + // or still-open group may have lost its tail. + const finished = first.done && !first.truncated; + const adjacent = next?.consumer.sequence === first.consumer.sequence + 1; + const continued = (finished && adjacent) || ptsContiguous(first.end ?? this.#presentedEnd, nextStart); + if (marker || discarded || first.truncated || !continued) this.#lost = true; first.consumer.close(); first.frames.length = 0; skipped++; diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 51102c62051..a83066c24af 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -239,11 +239,12 @@ impl Consumer { // first arrivals instead of assuming group 0. Skipping stays the budget's job. if self.startup { // NOTE: poll_min_timestamp buffers at least one frame per group and - // registers the waiter on the ones still empty. - let any_frame = self - .pending - .iter_mut() - .any(|group| matches!(group.poll_min_timestamp(waiter, &self.format), Poll::Ready(Ok(_)))); + // registers the waiter on the ones still empty. Any wire frame counts, so a + // paused track whose only group is a marker still reads its end. + let any_frame = self.pending.iter_mut().any(|group| { + let _ = group.poll_min_timestamp(waiter, &self.format); + !group.empty + }); if any_frame { self.current = self.pending.front().expect("a group has a frame").sequence; self.startup = false; @@ -963,6 +964,26 @@ mod tests { items } + /// Joining a paused track, whose only served group is a marker, reads the end without + /// waiting for media to resume. + #[tokio::test(start_paused = true)] + async fn a_join_on_a_marker_group_reads_its_end() { + let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscription = + track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); + write_marker_group(&mut track, 16, ts(1_000_000)); + + let mut read = async || { + tokio::time::timeout(Duration::from_secs(1), reader.read()) + .await + .expect("the end is read while the track stays open") + .unwrap() + }; + assert!(matches!(read().await, Some(Read::Gap))); + assert!(matches!(read().await, Some(Read::End { group: 16, end }) if end == ts(1_000_000))); + } + #[tokio::test] async fn gaps_mark_the_join_and_holes_while_a_marker_reads_as_an_end() { let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); From 0cb0cbcf3805b0d9c9670a2b935d87a864bf0a5f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 13 Oct 2026 11:08:18 -0700 Subject: [PATCH 18/19] fix(hang): read nothing after close, even the frame a gap was deferred ahead of Co-Authored-By: Claude Opus 5.5 --- js/hang/src/container/consumer.test.ts | 21 +++++++++++++++++++++ js/hang/src/container/consumer.ts | 2 ++ 2 files changed, 23 insertions(+) diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 8d9a9f8f4c1..412a6e3aca6 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -769,6 +769,27 @@ test("Consumer next() yields a gap before the first frame only", async () => { consumer.close(); }); +test("Consumer reads nothing after close, even the frame a gap was returned ahead of", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe(), { + format: new LegacyFormat("video"), + maxDelay: 500 as Time.Milli, + }); + + const group = new Group.Producer(0); + group.writeFrame({ + payload: encodeLegacyFrame(0 as Time.Micro, new Uint8Array([0xde, 0xad])), + timestamp: Time.Timestamp.now(), + }); + track.writeGroup(group); + + expect(await consumer.next()).toEqual({ kind: "gap" }); + consumer.close(); + expect(await consumer.next()).toBeUndefined(); + group.close(); + track.close(); +}); + test("Consumer keeps a trailing duration marker internal", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(track.subscribe(), { diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 7df377f186e..b0b7eb6b363 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -625,6 +625,8 @@ export class Consumer { /** Stop consuming and release the track and all buffered groups. */ close(): void { + // The item a gap was returned ahead of is buffered media too. + this.#next = undefined; this.#finish(null); this.#signals.close(); } From 1b38d59dfd9ddaed22d97f0edee043f87997090d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 13 Oct 2026 12:03:29 -0700 Subject: [PATCH 19/19] feat(mux): a gap carries its span, and a join is not a gap The container read's gap now says where the missing media starts (the furthest point already read) and how long until media resumes, as an upper bound, in both languages. Nothing read before the first item means no gap there. Watch's arrival gap carries the same span in milliseconds. Co-Authored-By: Claude Opus 5.5 --- doc/lib/js/watch.md | 12 +- doc/lib/rs/moq-mux.md | 6 +- js/hang/src/container/consumer.skips.test.ts | 1 - .../src/container/consumer.startup.test.ts | 27 ++-- js/hang/src/container/consumer.test.ts | 76 ++++++------ js/hang/src/container/consumer.ts | 50 +++++--- js/watch/src/arrivals.ts | 25 +++- js/watch/src/audio/decoder.test.ts | 4 +- js/watch/src/audio/decoder.ts | 6 +- js/watch/src/audio/replay.test.ts | 7 +- js/watch/src/audio/replay.ts | 4 +- js/watch/src/audio/terminal.test.ts | 2 +- js/watch/src/retention.test.ts | 2 +- js/watch/src/video/decoder.test.ts | 2 +- js/watch/src/video/decoder.ts | 6 +- quest/m1/audio-warmup.md | 5 +- quest/m1/open-gop-leading-pictures.md | 13 +- quest/m2/intra-refresh/consumer-warmup.md | 6 +- rs/hang/examples/subscribe.rs | 4 +- rs/moq-audio/src/decode/consumer.rs | 2 +- rs/moq-c/src/consume.rs | 2 +- rs/moq-cli/src/play/video.rs | 2 +- rs/moq-ffi/src/media.rs | 2 +- rs/moq-gst/src/source/imp.rs | 2 +- rs/moq-mux/src/container/consumer.rs | 115 +++++++++++++----- rs/moq-mux/src/container/mod.rs | 16 ++- rs/moq-rtc/src/codec/mod.rs | 2 +- rs/moq-srt/src/ts.rs | 2 +- rs/moq-video/src/decode/consumer.rs | 2 +- 29 files changed, 250 insertions(+), 155 deletions(-) diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index 0ec1b4aea1c..d58a887abc4 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -150,18 +150,24 @@ to change later, such as `muted` or `delay`. `Player` is the pipeline inside To chart when each frame arrived against when it plays, open an `arrivals()` handle on a decoder. It yields every frame received while it is open, stamped -when the frame came off the transport, and a gap wherever media will never -arrive (the join, a latency skip, a hole): +when the frame came off the transport, and a gap wherever media after an earlier +frame will never arrive (a latency skip, a hole): ```ts const arrivals = player.audio.arrivals(); for await (const entry of arrivals) { - if (entry.kind === "gap") continue; // break the plotted line here + if (entry.kind === "gap") { + // Break the plotted line; nothing arrived from `timestamp` for `duration`. + continue; + } const { timestamp, arrival, late } = entry; // `late`: arrived after its playback deadline. } ``` +A gap's span is an upper bound: without per-frame durations it starts at the +last frame's timestamp, and it covers any pause the publisher declared. + Unread frames queue on the handle, so keep reading and keep your own window. Leaving the loop or calling `arrivals.close()` stops observing and releases them; nothing is retained without an open handle. Video reports the rendition on screen. diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index 82255ac22ce..a29e578d10f 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -31,8 +31,10 @@ export. The [CLI page](/bin/cli#import) covers these and the MPEG-TS specifics: one program per broadcast, damaged packets, and feed checks. A container read yields frames plus two markers in delivery order: a gap where -media will never arrive (the join, a skip, a hole) and an end where the -publisher declared its media stops (a pause, a source switch, the finish). +media after what was read will never arrive (a skip, a hole), carrying where it +starts and how long it lasts as an upper bound, and an end where the publisher +declared its media stops (a pause, a source switch, the finish). Joining +mid-track is not a gap. Data tracks are catalog entries too. The [hang page](/concept/hang#data-tracks) describes the modes. A capture time converts onto the broadcast clock; a diff --git a/js/hang/src/container/consumer.skips.test.ts b/js/hang/src/container/consumer.skips.test.ts index 17447495cf3..cdcfe3c5c77 100644 --- a/js/hang/src/container/consumer.skips.test.ts +++ b/js/hang/src/container/consumer.skips.test.ts @@ -34,7 +34,6 @@ for (const count of [1, 16, 256]) { expect(warn).toHaveBeenLastCalledWith(`skipping slow groups: track=tone 0 -> ${count * 2} count=${count}`); expect(consumer.buffered.peek()[0]?.start).toBe(Time.Milli(count)); // The retained group is still delivered, and another catch-up reports separately. - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(await consumer.next()).toMatchObject({ kind: "frame", group: count * 2 }); newest.writeFrame({ payload: Varint.encode(count * 1000), timestamp: Time.Timestamp.now() }); write(count * 2 + 2, (count + 1) * 1000); diff --git a/js/hang/src/container/consumer.startup.test.ts b/js/hang/src/container/consumer.startup.test.ts index 5d8ecfacd8f..4d827ecac30 100644 --- a/js/hang/src/container/consumer.startup.test.ts +++ b/js/hang/src/container/consumer.startup.test.ts @@ -35,10 +35,10 @@ function setup(maxDelay = 100) { return { track, consumer, results, warn }; } -// Each item as "gap", "@<µs>" for a frame, or "end @<µs>". +// Each item as "gap <µs>+<µs>", "@<µs>" for a frame, or "end @<µs>". function items(results: Read[]): string[] { return results.map((read) => { - if (read.kind === "gap") return "gap"; + if (read.kind === "gap") return `gap ${read.timestamp}+${read.duration}`; if (read.kind === "end") return `end ${read.group}@${read.end}`; return `${read.group}@${read.frame.timestamp}`; }); @@ -79,8 +79,8 @@ for (const emptySuccessor of [false, true]) { for (let i = 0; i < 5; i++) write(live, 1_000_000 + i * 33_333); await flush(); + // Nothing was read before the live group, so the stale one leaves no gap. expect(items(results)).toEqual([ - "gap", ...[1_000_000, 1_033_333, 1_066_666, 1_099_999, 1_133_332].map( (timestamp) => `${sequence}@${timestamp}`, ), @@ -98,14 +98,15 @@ test("Consumer preserves a declared marker before its first media across a hole" write(marker, 0, true); marker.close(); await flush(); - expect(items(results)).toEqual(["gap", "end 16@0"]); + expect(items(results)).toEqual(["end 16@0"]); const live = new Group.Producer(18); track.writeGroup(live); write(live, 50_000); await flush(); - // Group 17 never arrived, so the media after the marker is past a gap. - expect(items(results)).toEqual(["gap", "end 16@0", "gap", "18@50000"]); + // Group 17 never arrived, so the media after the marker is past a gap. It spans the declared + // pause too: the reader can't tell where group 17 would have started. + expect(items(results)).toEqual(["end 16@0", "gap 0+50000", "18@50000"]); expect(frames(results)[0].frame.keyframe).toBe(true); expect(warn).not.toHaveBeenCalled(); }); @@ -117,7 +118,7 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(first, 0); first.close(); await flush(); - expect(items(results)).toEqual(["gap", "16@0"]); + expect(items(results)).toEqual(["16@0"]); if (reset) { const marker = new Group.Producer(17); @@ -125,7 +126,7 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(marker, 0, true); marker.close(); await flush(); - expect(items(results)).toEqual(["gap", "16@0", "end 17@0"]); + expect(items(results)).toEqual(["16@0", "end 17@0"]); } const sequence = reset ? 20 : 19; const later = new Group.Producer(sequence); @@ -141,13 +142,7 @@ test.each([false, true])("Consumer still waits for a gap after media (playhead r write(bridge, 0); bridge.close(); await flush(); - // The bridge closes the hole, so nothing reads as a gap past the first. - expect(items(results)).toEqual([ - "gap", - "16@0", - ...(reset ? ["end 17@0"] : []), - `${sequence - 1}@0`, - `${sequence}@50000`, - ]); + // The bridge closes the hole, so nothing reads as a gap. + expect(items(results)).toEqual(["16@0", ...(reset ? ["end 17@0"] : []), `${sequence - 1}@0`, `${sequence}@50000`]); expect(warn).not.toHaveBeenCalled(); }); diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index 412a6e3aca6..5d6e1e2ae0e 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -529,9 +529,9 @@ async function drainFrames( return frames; } -/** An item as "gap", a frame's µs, or "end <µs>". */ +/** An item as "gap <µs>+<µs>", a frame's µs, or "end <µs>". */ function describe(read: Read): string { - if (read.kind === "gap") return "gap"; + if (read.kind === "gap") return `gap ${read.timestamp}+${read.duration}`; if (read.kind === "end") return `end ${read.end}`; return `${read.frame.timestamp}`; } @@ -745,7 +745,7 @@ test("Consumer delivers a group that arrives below the cursor", async () => { // --- Group.Producer boundary signals --- -test("Consumer next() yields a gap before the first frame only", async () => { +test("Consumer next() yields no gap before the first frame", async () => { const track = new Track.Producer("test"); const consumer = new Consumer(replay(track), { format: new LegacyFormat("data"), maxDelay: 500 as Time.Milli }); @@ -765,28 +765,25 @@ test("Consumer next() yields a gap before the first frame only", async () => { seen.push(describe(result)); } - expect(seen).toEqual(["gap", "0", "33000", "66000"]); + expect(seen).toEqual(["0", "33000", "66000"]); consumer.close(); }); test("Consumer reads nothing after close, even the frame a gap was returned ahead of", async () => { const track = new Track.Producer("test"); - const consumer = new Consumer(track.subscribe(), { - format: new LegacyFormat("video"), - maxDelay: 500 as Time.Milli, - }); + const consumer = new Consumer(replay(track), { format: new LegacyFormat("audio"), maxDelay: 0 as Time.Milli }); - const group = new Group.Producer(0); - group.writeFrame({ - payload: encodeLegacyFrame(0 as Time.Micro, new Uint8Array([0xde, 0xad])), - timestamp: Time.Timestamp.now(), - }); - track.writeGroup(group); + writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); + await settle(); + expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + + // Group 1 never arrives, so group 2's frame is deferred behind a gap. + writeGroupWithLegacyFrames(track, 2, [100_000 as Time.Micro]); + await settle(); + expect(await consumer.next()).toEqual({ kind: "gap", timestamp: 0 as Time.Micro, duration: 100_000 as Time.Micro }); - expect(await consumer.next()).toEqual({ kind: "gap" }); consumer.close(); expect(await consumer.next()).toBeUndefined(); - group.close(); track.close(); }); @@ -811,7 +808,6 @@ test("Consumer keeps a trailing duration marker internal", async () => { track.close(); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); const media = frameOf(await consumer.next()); expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); expect(media.keyframe).toBe(true); @@ -841,7 +837,6 @@ test("Consumer reads a leading marker as an end and keeps the first media keyfra track.writeGroup(group); track.close(); - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(await consumer.next()).toEqual({ kind: "end", group: 0, end: 20_000 as Time.Micro }); const media = frameOf(await consumer.next()); expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); @@ -859,7 +854,6 @@ test("Consumer reads an empty LOC payload as the frame's duration", async () => const consumer = new Consumer(replay(track), { format: new LocFormat("video"), maxDelay: 500 as Time.Milli }); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); const media = frameOf(await consumer.next()); expect(media.payload).toEqual(new Uint8Array([0xde, 0xad])); expect(media.duration).toBeUndefined(); @@ -877,7 +871,6 @@ test("Consumer passes over the duration markers the LOC producer writes", async const consumer = new Consumer(replay(track), { format: new LocFormat("video"), maxDelay: 500 as Time.Milli }); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); const first = frameOf(await consumer.next()); expect(first.payload).toEqual(new Uint8Array([0xde, 0xad])); expect(first.keyframe).toBe(true); @@ -898,7 +891,6 @@ test("Consumer preserves empty Legacy and LOC data frames", async () => { producer.close(); const format = kind === "legacy" ? new LegacyFormat("data") : new LocFormat(); const consumer = new Consumer(replay(track), { format, maxDelay: 500 as Time.Milli }); - expect(await consumer.next()).toEqual({ kind: "gap" }); const frame = frameOf(await consumer.next()); expect(frame.payload).toEqual(new Uint8Array()); expect(frame.keyframe).toBe(true); @@ -955,7 +947,6 @@ test("Consumer treats B-frame reordering within a group as continuous", async () writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 66_000 as Time.Micro, 33_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); for (const timestamp of [0, 66_000, 33_000]) { expect(frameOf(await consumer.next()).timestamp).toBe(timestamp as Time.Micro); } @@ -978,7 +969,6 @@ test("Consumer accepts open-GOP leading pictures above the previous group", asyn writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 33_000 as Time.Micro]); writeGroupWithLegacyFrames(track, 1, [66_000 as Time.Micro, 50_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); for (const timestamp of [0, 33_000, 66_000, 50_000]) { expect(frameOf(await consumer.next()).timestamp).toBe(timestamp as Time.Micro); } @@ -994,7 +984,6 @@ test("Consumer reads no gap when retiring a fully delivered group", async () => writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); @@ -1021,7 +1010,6 @@ test("Consumer reads no gap when retiring a group holding only its endpoint", as group.close(); track.writeGroup(group); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); @@ -1043,7 +1031,6 @@ test("Consumer reads a gap when retiring a truncated group", async () => { group.writeFrame({ payload: encodeLegacy(0 as Time.Micro), timestamp: Time.Timestamp.now() }); track.writeGroup(group); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); group.close(new NetError.Stream(StreamCode.Cancel)); @@ -1051,7 +1038,8 @@ test("Consumer reads a gap when retiring a truncated group", async () => { writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); + // Legacy carries no duration, so the gap starts at the last frame read. + expect(await consumer.next()).toEqual({ kind: "gap", timestamp: 0 as Time.Micro, duration: 100_000 as Time.Micro }); expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); consumer.close(); @@ -1066,14 +1054,14 @@ test("Consumer reads a gap when retiring an open group", async () => { group.writeFrame({ payload: encodeLegacy(0 as Time.Micro), timestamp: Time.Timestamp.now() }); track.writeGroup(group); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); writeGroupWithLegacyFrames(track, 1, [100_000 as Time.Micro]); writeGroupWithLegacyFrames(track, 2, [200_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); + // Legacy carries no duration, so the gap starts at the last frame read. + expect(await consumer.next()).toEqual({ kind: "gap", timestamp: 0 as Time.Micro, duration: 100_000 as Time.Micro }); expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); group.close(); @@ -1543,9 +1531,9 @@ test("Consumer plays the head once a waited-out gap exceeds the budget (CMAF)", track.writeGroup(b); b.writeFrame(cmafFrame(0x02, 90_000, 1)); - // The missing group's span was never delivered, so a gap comes first. + // The missing group's span was never delivered, so a gap from A's end to B comes first. const result = await Promise.race([pending, settle(300).then(() => "timeout" as const)]); - expect(result).toEqual({ kind: "gap" }); + expect(result).toEqual({ kind: "gap", timestamp: 33_333 as Time.Micro, duration: 966_667 as Time.Micro }); expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); consumer.close(); @@ -1582,7 +1570,7 @@ test("Consumer waits on a gap within the budget, then plays the head once it exc } const result = await Promise.race([pending, settle(300).then(() => "timeout" as const)]); - expect(result).toEqual({ kind: "gap" }); + expect(result).toEqual({ kind: "gap", timestamp: 33_333 as Time.Micro, duration: 33_334 as Time.Micro }); expect(frameOf(await consumer.next()).payload).toEqual(new Uint8Array([0x02])); consumer.close(); @@ -1600,7 +1588,6 @@ test("Consumer yields no gap while nothing is dropped", async () => { writeGroupWithLegacyFrames(track, 0, [0 as Time.Micro, 33_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); // nothing precedes the first frame expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); expect(frameOf(await consumer.next()).timestamp).toBe(33_000 as Time.Micro); @@ -1620,7 +1607,6 @@ test("Consumer reads a marker group as an end, not a gap", async () => { writeGroupWithLegacyFrames(track, 2, [1_000_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); expect(await consumer.next()).toEqual({ kind: "end", group: 1, end: 0 as Time.Micro }); // A declared stop loses nothing, so no gap follows it. @@ -1640,7 +1626,6 @@ test("Consumer empty groups mean nothing", async () => { writeGroupWithLegacyFrames(track, 2, [1_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); expect(frameOf(await consumer.next()).timestamp).toBe(1_000 as Time.Micro); @@ -1658,7 +1643,7 @@ test("Consumer reads a latency skip as one gap", async () => { writeGroupWithLegacyFrames(track, 2, [100_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(await consumer.next()).toEqual({ kind: "gap", timestamp: 0 as Time.Micro, duration: 100_000 as Time.Micro }); expect(frameOf(await consumer.next()).timestamp).toBe(100_000 as Time.Micro); consumer.close(); @@ -1675,7 +1660,11 @@ test("Consumer reads a timestamp hole as a gap", async () => { writeGroupWithLegacyFrames(track, 3, [1_000_000 as Time.Micro]); await settle(); - expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(await consumer.next()).toEqual({ + kind: "gap", + timestamp: 0 as Time.Micro, + duration: 1_000_000 as Time.Micro, + }); expect(frameOf(await consumer.next()).timestamp).toBe(1_000_000 as Time.Micro); consumer.close(); @@ -1699,7 +1688,7 @@ test("Consumer reads a shed marker group as a gap", async () => { await settle(); // The zero budget sheds the marker unread, and what it declared is lost with it. - expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(await consumer.next()).toEqual({ kind: "gap", timestamp: 0 as Time.Micro, duration: 500 as Time.Micro }); expect(frameOf(await consumer.next()).timestamp).toBe(500 as Time.Micro); group0.close(); @@ -1749,8 +1738,7 @@ test("a live video frame is delivered before its duration marker arrives", async }); await settle(); expect(delivered).toBe(true); - expect(await read).toEqual({ kind: "gap" }); - expect(frameOf(await consumer.next()).timestamp).toBe(0 as Time.Micro); + expect(frameOf(await read).timestamp).toBe(0 as Time.Micro); group.writeFrame({ payload: encodeLegacyFrame(15_000 as Time.Micro, new Uint8Array()), timestamp: Time.Timestamp.now(), @@ -1797,7 +1785,11 @@ for (const code of [StreamCode.Cancel, StreamCode.Internal, StreamCode.Old, Stre }); next.close(); // The reset cut the group's tail. - expect(await consumer.next()).toEqual({ kind: "gap" }); + expect(await consumer.next()).toEqual({ + kind: "gap", + timestamp: 0 as Time.Micro, + duration: 1 as Time.Micro, + }); const resumed = frameOf(await consumer.next()); expect(resumed.payload).toEqual(new Uint8Array([2])); expect(resumed.keyframe).toBe(true); @@ -1872,7 +1864,7 @@ for (const end of [ const resumed = await Promise.race([pending, settle(100).then(() => "stalled" as const)]); expect(resumed).not.toBe("stalled"); if (resumed === "stalled") return; - expect(resumed).toEqual({ kind: "gap" }); + expect(resumed).toEqual({ kind: "gap", timestamp: 0 as Time.Micro, duration: 1_000_000 as Time.Micro }); expect(await consumer.next()).toMatchObject({ kind: "frame", group: 2, frame: { timestamp: 1_000_000 } }); if (end) await expect(consumer.next()).rejects.toBe(end); else expect(await consumer.next()).toBeUndefined(); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index b0b7eb6b363..65852712752 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -26,11 +26,15 @@ export type Read = /** A media frame from `group`. Only group equality means anything: numbers may skip. */ | { kind: "frame"; group: number; frame: ReceivedFrame } /** - * Media before the next item will never reach this reader: it joined mid-track, or a group was - * skipped, evicted, truncated, or never arrived. A decoder resumes without the references or - * state that media built. + * Media after an item already read will never reach this reader: a group was skipped, evicted, + * truncated, or never arrived. A decoder resumes without the references or state that media + * built. Joining mid-track is not a gap: nothing was read before it. + * + * The span is an upper bound. `timestamp` is the furthest point already read; without a + * per-frame duration that is the last frame's timestamp, one frame early. `duration` runs to + * where the next item resumes, so it also covers any declared pause in between. */ - | { kind: "gap" } + | { kind: "gap"; timestamp: Time.Micro; duration: Time.Micro } /** * The publisher declared that its media stops at `end`: it paused, switched source, or finished. * Frames that follow from the same `group` are the stopped encoder's final packets (an audio @@ -98,9 +102,11 @@ export class Consumer { // Group of the last media frame next() returned, so an endpoint after it reads as that frame's // duration rather than a declared stop. #mediaGroup?: number; - // Media went missing since the last item read, so the next one is preceded by a gap. Starts set: - // nothing before the first frame was read. - #lost = true; + // Media went missing since the last item read, so the next one is preceded by a gap. + #lost = false; + // The furthest presentation point read so far: max(timestamp + duration), endpoints included. + // Undefined until the first item, so a loss before it is no gap. + #edge?: Time.Micro; // The item a gap was returned ahead of. #next?: Read; // The first media timestamp of the latest delivered group, and its predecessor's start. @@ -450,7 +456,7 @@ export class Consumer { /** * Returns the next {@link Read} in delivery order, awaiting one if needed: a frame, a gap where - * media will never arrive (always one before the first frame), or an end the publisher declared. + * media after an earlier item will never arrive, or an end the publisher declared. * An unmarked forward timestamp jump is not a gap, since nothing on the wire says media is missing. * After buffered groups drain, a finished track returns undefined and an aborted track throws. */ @@ -518,8 +524,11 @@ export class Consumer { const end = this.#format.end?.(frame); if (end !== undefined) { this.#updateBuffered(); - // A trailing endpoint is the last frame's duration, which only the skip logic needs. - if (seq === this.#mediaGroup) continue; + // A trailing endpoint is the last frame's duration: it moves the edge, nothing else. + if (seq === this.#mediaGroup) { + this.#advance(end); + continue; + } return this.#emit({ kind: "end", group: seq, end }); } this.#mediaGroup = seq; @@ -589,12 +598,25 @@ export class Consumer { } } - // Return `read`, or a gap ahead of it when media went missing first. - #emit(read: Read): Read { - if (!this.#lost) return read; + // Return `read`, or a gap ahead of it when media went missing after an earlier item. + #emit(read: Exclude): Read { + const resume = read.kind === "frame" ? read.frame.timestamp : read.end; + const edge = this.#edge; + const lost = this.#lost; this.#lost = false; + this.#advance( + read.kind === "frame" ? Moq.Time.Micro.add(resume, read.frame.duration ?? Moq.Time.Micro.zero) : resume, + ); + if (!lost || edge === undefined) return read; + + // A next group may start before the edge (an overlapping endpoint), so the span bottoms out at zero. + const timestamp = Moq.Time.Micro.min(edge, resume); this.#next = read; - return { kind: "gap" }; + return { kind: "gap", timestamp, duration: Moq.Time.Micro.sub(resume, timestamp) }; + } + + #advance(end: Time.Micro): void { + if (this.#edge === undefined || end > this.#edge) this.#edge = end; } #updateBuffered(): void { diff --git a/js/watch/src/arrivals.ts b/js/watch/src/arrivals.ts index d4fba3df729..ecb1b75d236 100644 --- a/js/watch/src/arrivals.ts +++ b/js/watch/src/arrivals.ts @@ -1,4 +1,5 @@ -import type { Time } from "@moq/net"; +import type * as Container from "@moq/hang/container"; +import { Time } from "@moq/net"; /** One received frame, or a gap where media will never arrive, in delivery order. */ export type Arrival = @@ -11,8 +12,26 @@ export type Arrival = /** Whether ingress missed the playback deadline. */ late: boolean; } - /** Media before the next frame never reached the decoder: the join, a latency skip, or a hole. */ - | { kind: "gap" }; + /** + * Media after an earlier frame never reached the decoder: a latency skip or a hole. The span is + * an upper bound, as on the container read's gap. + */ + | { + kind: "gap"; + /** The furthest presentation point received before the gap, in milliseconds. */ + timestamp: Time.Milli; + /** How long until media resumes, in milliseconds. */ + duration: Time.Milli; + }; + +/** The arrival entry for a container read's gap. */ +export function gap(read: Extract): Arrival { + return { + kind: "gap", + timestamp: Time.Milli.fromMicro(read.timestamp), + duration: Time.Milli.fromMicro(read.duration), + }; +} interface State { queue: Arrival[]; diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 325e2ad584d..c4b42779e7b 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -302,13 +302,13 @@ it("decodes an audio packet inside a group after a gap", async () => { await playback.play(); // Opened late, so it observes only the gap and frame below. const arrivals = playback.arrivals(); - playback.push({ kind: "gap" }); + playback.push({ kind: "gap", timestamp: Time.Micro(20_000), duration: Time.Micro(40_000) }); const next = frame(); next.frame.keyframe = false; playback.push(next); await microtasks(); expect(playback.inserted()).toContain(next.frame.timestamp); - expect(await arrivals.next()).toEqual({ kind: "gap" }); + expect(await arrivals.next()).toEqual({ kind: "gap", timestamp: Time.Milli(20), duration: Time.Milli(40) }); expect(await arrivals.next()).toMatchObject({ kind: "frame", timestamp: Time.Milli.fromMicro(next.frame.timestamp), diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 64a926468e3..fee44b1e310 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -13,7 +13,7 @@ import { readonlys, Signal, } from "@moq/signals"; -import { Producer as ArrivalProducer, type Arrivals } from "../arrivals"; +import { Producer as ArrivalProducer, type Arrivals, gap } from "../arrivals"; import { hostedAssets } from "../assets"; import { base64ToBytes } from "../base64"; import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media"; @@ -479,7 +479,7 @@ export class Decoder { decoder.reset(); decoder.configure(decoderConfig); } - if (next.kind === "gap") this.#arrivals.write({ kind: "gap" }); + if (next.kind === "gap") this.#arrivals.write(gap(next)); if (next.kind !== "frame") continue; const { frame } = next; @@ -587,7 +587,7 @@ export class Decoder { decoder.reset(); decoder.configure(decoderConfig); } - if (next.kind === "gap") this.#arrivals.write({ kind: "gap" }); + if (next.kind === "gap") this.#arrivals.write(gap(next)); if (next.kind !== "frame") continue; const { frame } = next; diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts index d39ba7f4534..1f5e70d2bed 100644 --- a/js/watch/src/audio/replay.test.ts +++ b/js/watch/src/audio/replay.test.ts @@ -154,14 +154,13 @@ describe.each(["shared", "post"] as const)("%s ingress diagnostics", (ring) => { const direct = await run(); const held = await run(1200); expect(held).toEqual(direct); - expect(held).toEqual([ - { kind: "gap" }, - ...trace.map((entry) => ({ + expect(held).toEqual( + trace.map((entry) => ({ kind: "frame" as const, timestamp: Time.Milli(entry.timestamp), arrival: Time.Milli(entry.at), late: false, })), - ]); + ); }); }); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index 43b7a3de2dd..fff6d602d8a 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -19,7 +19,7 @@ import type * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; import { Group, Time, Track } from "@moq/net"; import { Effect } from "@moq/signals"; -import type { Arrival as Diagnostic } from "../arrivals"; +import { type Arrival as Diagnostic, gap } from "../arrivals"; import { type Delay, Sync } from "../sync"; import { frameDuration } from "./config"; import { ringSamples, target } from "./latency"; @@ -230,7 +230,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const next = await consumer.next(); if (!next) return; if (terminal.update(next)) ring.reset(); - if (next.kind === "gap") arrivals.push({ kind: "gap" }); + if (next.kind === "gap") arrivals.push(gap(next)); if (next.kind !== "frame") continue; const { timestamp, arrival } = next.frame; diff --git a/js/watch/src/audio/terminal.test.ts b/js/watch/src/audio/terminal.test.ts index 44eb0d06c33..f766ef247e0 100644 --- a/js/watch/src/audio/terminal.test.ts +++ b/js/watch/src/audio/terminal.test.ts @@ -39,7 +39,7 @@ test("Terminal preserves samples before the source endpoint", () => { expect(terminal.span({ timestamp: 0, sampleRate: 48_000, numberOfFrames: 960 }).frames).toBe(960); }); -test("the gap before the first frame ends no epoch", () => { +test("a gap before any frame ends no epoch", () => { const terminal = new Terminal(); expect(terminal.update(gap)).toBe(false); }); diff --git a/js/watch/src/retention.test.ts b/js/watch/src/retention.test.ts index 457b2ec975d..d8b498827d9 100644 --- a/js/watch/src/retention.test.ts +++ b/js/watch/src/retention.test.ts @@ -100,7 +100,7 @@ test("a long subscription through the player path leaves nothing behind", async const play = async (count: number) => { for (let i = 0; i < count; i++) { producer.encode(new Uint8Array([i & 0xff]), Math.round(Time.Micro.now()) as Time.Micro, true); - // Pass over the gap before the first frame. + // Pass over any gap or end. let next = await nextMedia(consumer); while (next && next.kind !== "frame") next = await nextMedia(consumer); if (!next) throw new Error("the track ended"); diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index 35e0f8efea2..194901f5c47 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -181,7 +181,7 @@ function sample(group: number, timestamp: number, keyframe = true): Read { frame: { arrival: Time.Milli.now(), timestamp: Time.Micro(timestamp), payload: new Uint8Array([1]), keyframe }, }; } -const gap: Read = { kind: "gap" }; +const gap: Read = { kind: "gap", timestamp: Time.Micro(0), duration: Time.Micro(0) }; async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: readonly number[] = []) { const originalDecoder = Object.getOwnPropertyDescriptor(globalThis, "VideoDecoder"); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 55983a2bccb..cc99c163c9f 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -14,7 +14,7 @@ import { Signal, } from "@moq/signals"; import { rotateVideoDimensions } from "@moq/video"; -import { Producer as ArrivalProducer, type Arrivals } from "../arrivals"; +import { Producer as ArrivalProducer, type Arrivals, gap } from "../arrivals"; import { base64ToBytes } from "../base64"; import { type Failure, failureMoved, nextMedia, subscribeMedia } from "../media"; @@ -562,7 +562,7 @@ class DecoderTrack { // A gap or a declared end breaks the timeline: re-anchor before decoding further, and // never bridge the buffer across it. if (next.kind !== "frame") { - if (next.kind === "gap") this.arrivals?.write({ kind: "gap" }); + if (next.kind === "gap") this.arrivals?.write(gap(next)); if (latest !== undefined) this.#onBreak(); previous = undefined; continue; @@ -646,7 +646,7 @@ class DecoderTrack { // A gap or a declared end breaks the timeline (see #runLegacy). if (next.kind !== "frame") { - if (next.kind === "gap") this.arrivals?.write({ kind: "gap" }); + if (next.kind === "gap") this.arrivals?.write(gap(next)); if (latest !== undefined) this.#onBreak(); previous = undefined; continue; diff --git a/quest/m1/audio-warmup.md b/quest/m1/audio-warmup.md index 1f8888b360a..4f334674b83 100644 --- a/quest/m1/audio-warmup.md +++ b/quest/m1/audio-warmup.md @@ -14,8 +14,9 @@ frames decode independently and set nothing; HE-AAC is out of scope. - `rs/moq-audio/src/encode/producer.rs` and `rs/moq-mux/src/codec/opus` publish `warmup` for Opus; `js/publish` does the same for its Opus track. - `rs/moq-audio/src/decode` already trims Opus `pre_skip` at stream start; - the warmup trim is the same mechanism keyed on the container read's gap - item (`container::Read::Gap` in Rust, `{ kind: "gap" }` in JS), never on a + the warmup trim is the same mechanism keyed on the first frame read and the + container read's gap item (`container::Read::Gap` in Rust, `{ kind: "gap" }` + in JS; a join yields no gap), never on a declared end, which would cut 80 ms at every pause, and the subscription's maximum age grows by `warmup` as the video consumer quest does. `js/watch` audio mirrors it. diff --git a/quest/m1/open-gop-leading-pictures.md b/quest/m1/open-gop-leading-pictures.md index fa317d9f3c6..7c2f8526cc5 100644 --- a/quest/m1/open-gop-leading-pictures.md +++ b/quest/m1/open-gop-leading-pictures.md @@ -38,16 +38,16 @@ decode, and the JS test should drive the software decoder. - In the JS consumer (`js/hang/src/container/consumer.ts` forces the first sample of a group to `keyframe`, and `js/watch/src/video/decoder.ts` submits - it as `"key"`): for the first group after a gap item, skip delta frames - stamped before that group's keyframe. `next()` yields `{ kind: "gap" }` - before the first frame of a subscribe and after a latency skip, eviction, or - hole; a declared end is not a gap, since a fresh encoder has no earlier GOP + it as `"key"`): for the first group read and the first group after a gap + item, skip delta frames stamped before that group's keyframe. `next()` + yields `{ kind: "gap" }` after a latency skip, eviction, or hole, but not + before the first frame of a subscribe; a declared end is not a gap, since a fresh encoder has no earlier GOP to reference. A latency skip does not flush the decoder, and a viewer that skipped into a later GOP lacks its references just like a cold join. Every group without a gap before it is passed through untouched. - The same rule in the Rust decode path (`moq-video` decode consumers), so native playback and the transcoder tune in the same way, keyed on the - container read's `Read::Gap` item. + first frame read and the container read's `Read::Gap` item. - The same rule in `moq export ts`. Decided (2026-10-01): the fixed-delay export still sends a join's orphaned leading pictures, so 3 of 500 frames on the open-GOP fixture decode after they present @@ -71,7 +71,8 @@ decode, and the JS test should drive the software decoder. the initial group fails. Both cases in both languages. Decided 2026-10-12: both container reads yield the gap item, so nothing -here waits on another quest. +here waits on another quest. Decided 2026-10-13 (#5311): a join is not a gap, +so the first frame read is its own trigger. ## Related diff --git a/quest/m2/intra-refresh/consumer-warmup.md b/quest/m2/intra-refresh/consumer-warmup.md index 0ad9d0e0ffa..22fe2f67e3d 100644 --- a/quest/m2/intra-refresh/consumer-warmup.md +++ b/quest/m2/intra-refresh/consumer-warmup.md @@ -19,12 +19,12 @@ This absorbs the old gradual-recovery quest, which planned to carry `recovery_frame_cnt` on the wire and count `frame_num`. The catalog duration replaces both: the rule is timestamp arithmetic on the group start. -- Withhold rule, keyed on the gap item both container reads yield before the - first frame and after any skip: `{ kind: "gap" }` from +- Withhold rule, keyed on the first frame a reader reads and on the gap item + both container reads yield after any skip (a join yields no gap): `{ kind: "gap" }` from `js/hang/src/container/consumer.ts` `next()` and `Read::Gap` from the Rust `moq_mux::container::Consumer`, which `rs/moq-video/src/decode/consumer.rs` reads directly. A declared end is not a gap and keeps everything. For the - first group after a gap, every frame is decoded (the decoder needs them to + first group read and the first group after a gap, every frame is decoded (the decoder needs them to build reference state) and frames stamped below `group.start + warmup` are not presented; frames stamped at or above that boundary are, so the recovery picture itself is never withheld. `js/watch/src/video/decoder.ts` and diff --git a/rs/hang/examples/subscribe.rs b/rs/hang/examples/subscribe.rs index 2e1226bdbf9..8b74ba453d8 100644 --- a/rs/hang/examples/subscribe.rs +++ b/rs/hang/examples/subscribe.rs @@ -123,7 +123,9 @@ async fn play(consumer: moq_net::origin::Consumer, announce: moq_net::announce:: bytes = frame.payload.len(), "received frame" ), - moq_mux::container::Read::Gap => tracing::info!("media skipped"), + moq_mux::container::Read::Gap { timestamp, duration } => { + tracing::info!(?timestamp, ?duration, "media skipped") + } moq_mux::container::Read::End { end, .. } => tracing::info!(?end, "media stopped"), } } diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 0e34a54c344..ee247471059 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -278,7 +278,7 @@ impl Consumer { } frame } - Some(Read::Gap) => { + Some(Read::Gap { .. }) => { self.restart()?; continue; } diff --git a/rs/moq-c/src/consume.rs b/rs/moq-c/src/consume.rs index e17afe2f62d..a2717454037 100644 --- a/rs/moq-c/src/consume.rs +++ b/rs/moq-c/src/consume.rs @@ -542,7 +542,7 @@ impl Consume { read = track.read() => match read? { Some(moq_mux::container::Read::Frame { frame, .. }) => frame, // The C ABI carries frames only. - Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. }) => continue, + Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. }) => continue, None => return Ok(()), }, }; diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 35a0cc608e5..6bc0487bcba 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -98,7 +98,7 @@ impl Video { false } // Applied in delivery order, so the restart lands before the frame after it. - Some(Read::Gap) => { + Some(Read::Gap { .. }) => { end = None; buffer.clear(); self.presentation.lock().unwrap().video_restarted(); diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 6782c05fc62..de6ca3af1c4 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -1115,7 +1115,7 @@ impl Media { match self.inner.read().await? { Some(moq_mux::container::Read::Frame { frame, .. }) => return media_frame(frame).map(Some), // The bindings carry frames only. - Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. }) => continue, + Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. }) => continue, None => return Ok(None), } } diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 4ae77a68b82..7f01e145c21 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -1099,7 +1099,7 @@ impl Pump { // This rendition is being torn down (its run ended, or a catalog update replaced it). _ = cancel.changed() => return None, frame = track.read() => match frame { - Ok(Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. })) => {} + Ok(Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. })) => {} Ok(Some(moq_mux::container::Read::Frame { frame, .. })) => { // PTS restarts at zero with every pump, so the segment places that zero at the // current running time, or a synced sink would drop a restarted run as late. diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index a83066c24af..b10bd4aba86 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -56,9 +56,10 @@ pub(crate) enum Event { /// /// ## Gaps and ends /// -/// [`read`](Self::read) yields [`Read::Gap`] before the first frame and wherever media +/// [`read`](Self::read) yields [`Read::Gap`] wherever media after an item already read /// will never arrive: a latency skip, an evicted current group, or a sequence hole whose -/// boundary is not contiguous within 1 ms. A marker group (no decodable frames, one empty +/// boundary is not contiguous within 1 ms. Whatever is lost before the first item is no +/// gap, since nothing was read before it. A marker group (no decodable frames, one empty /// payload) is the publisher declaring a stop, read as [`Read::End`]; nothing is missing. /// Empty groups (zero objects) mean nothing. A group whose timestamps fall below the live /// edge earlier groups reached is malformed and aborts the track. @@ -80,9 +81,9 @@ pub struct Consumer { // How far we may drift from the live edge before skipping a group. max_delay: std::time::Duration, - // The live edge of playback: the largest timestamp delivered so far and the group that - // carried it. `None` until the first frame is delivered. - live_edge: Option<(u64, Timestamp)>, + // The furthest presentation point delivered so far: max(timestamp + duration), endpoints + // included. `None` until the first item, so a loss before it is no gap. + edge: Option, // The first frame delivered from the latest group, and that group. Group starts never go // backwards, so a later group with a frame below this is malformed. @@ -104,7 +105,6 @@ pub struct Consumer { markers: u64, // Media went missing since the last item read, so the next one is preceded by a gap. - // Starts set: nothing before the first frame was read. gap: bool, // The item a gap was returned ahead of. @@ -124,6 +124,22 @@ fn pts_contiguous(end: Option, next_start: std::time::Durat end.is_some_and(|end| next_start <= end.saturating_add(CONTIGUITY_TOLERANCE)) } +/// A gap from the delivered `edge` to `resume`, in the resume's scale. +fn gap(edge: std::time::Duration, resume: Timestamp) -> Read { + let scale = resume.scale(); + // A next group may start before the edge (an overlapping endpoint), so the span bottoms out at zero. + let start = edge + .as_nanos() + .saturating_mul(scale.as_u64() as u128) + .div_euclid(1_000_000_000) + .min(resume.value() as u128) as u64; + let timestamp = Timestamp::new(start, scale).expect("at or below a valid timestamp"); + let duration = resume + .checked_sub(timestamp) + .expect("at or below the resume, in its scale"); + Read::Gap { timestamp, duration } +} + impl Consumer { /// Create a Consumer wrapping the given moq-lite consumer, decoding `format`. /// @@ -150,13 +166,13 @@ impl Consumer { pending: VecDeque::new(), startup: start.is_none(), max_delay, - live_edge: None, + edge: None, start: None, floor: None, presented_end: None, discontinuity: 0, markers: 0, - gap: true, + gap: false, next: None, group: 0, } @@ -194,7 +210,7 @@ impl Consumer { loop { match self.read().await? { Some(Read::Frame { frame, .. }) => return Ok(Some(frame)), - Some(Read::Gap | Read::End { .. }) => continue, + Some(Read::Gap { .. } | Read::End { .. }) => continue, None => return Ok(None), } } @@ -209,6 +225,8 @@ impl Consumer { return Poll::Ready(Ok(Some(read))); } loop { + // Where delivered media reached before this item, which moves the edge. + let edge = self.edge; let read = match ready!(self.poll_event(waiter))? { Some(Event::Frame(frame)) => Read::Frame { group: self.group, @@ -219,9 +237,16 @@ impl Consumer { Some(Event::FrameEnd { opens: false, .. } | Event::GroupEnd) => continue, None => return Poll::Ready(Ok(None)), }; - if std::mem::take(&mut self.gap) { + let resume = match &read { + Read::Frame { frame, .. } => frame.timestamp, + Read::End { end, .. } => *end, + Read::Gap { .. } => unreachable!("events never read as a gap"), + }; + if std::mem::take(&mut self.gap) + && let Some(edge) = edge + { self.next = Some(read); - return Poll::Ready(Ok(Some(Read::Gap))); + return Poll::Ready(Ok(Some(gap(edge, resume)))); } return Poll::Ready(Ok(Some(read))); } @@ -282,21 +307,15 @@ impl Consumer { if self.start.is_none_or(|(start, _)| start != seq) { self.start = Some((seq, ts)); } - if self.live_edge.is_none_or(|(_, high)| ts.as_micros() > high.as_micros()) { - self.live_edge = Some((seq, ts)); - } + let end = std::time::Duration::from(ts) + + frame.duration.map(std::time::Duration::from).unwrap_or_default(); + self.advance(end); self.group = seq; return Poll::Ready(Ok(Some(Event::Frame(frame)))); } Poll::Ready(Ok(Some(Event::FrameEnd { end, opens }))) => { - let seq = group.group.sequence; - if self - .live_edge - .is_none_or(|(_, high)| end.as_micros() > high.as_micros()) - { - self.live_edge = Some((seq, end)); - } - self.group = seq; + self.group = group.group.sequence; + self.advance(end.into()); return Poll::Ready(Ok(Some(Event::FrameEnd { end, opens }))); } Poll::Ready(Ok(Some(event))) => return Poll::Ready(Ok(Some(event))), @@ -489,6 +508,12 @@ impl Consumer { self.gap = true; } + fn advance(&mut self, end: std::time::Duration) { + if self.edge.is_none_or(|edge| end > edge) { + self.edge = Some(end); + } + } + fn note_group_edge(&mut self) { if let Some((_, ts)) = self.start { self.floor = Some(ts); @@ -951,13 +976,14 @@ mod tests { group.finish().unwrap(); } - /// Every item until the track ends: a frame as its timestamp in µs, a gap, or an end. + /// Every item until the track ends: a frame as its timestamp in µs, a gap as its start and + /// span in µs, or an end. async fn items(consumer: &mut Consumer) -> Vec { let mut items = Vec::new(); while let Some(read) = consumer.read().await.unwrap() { items.push(match read { Read::Frame { frame, .. } => frame.timestamp.as_micros().to_string(), - Read::Gap => "gap".to_string(), + Read::Gap { timestamp, duration } => format!("gap {}+{}", timestamp.as_micros(), duration.as_micros()), Read::End { end, .. } => format!("end {}", end.as_micros()), }); } @@ -980,12 +1006,31 @@ mod tests { .expect("the end is read while the track stays open") .unwrap() }; - assert!(matches!(read().await, Some(Read::Gap))); assert!(matches!(read().await, Some(Read::End { group: 16, end }) if end == ts(1_000_000))); } + /// A gap runs from the delivered edge to the resume point, in the resume's scale, and + /// never past it. + #[test] + fn a_gap_spans_the_edge_to_the_resume_in_its_scale() { + let scale = moq_net::Timescale::new(90_000).unwrap(); + let resume = Timestamp::new(90_000, scale).unwrap(); + let Read::Gap { timestamp, duration } = gap(Duration::from_millis(500), resume) else { + unreachable!() + }; + assert_eq!(timestamp, Timestamp::new(45_000, scale).unwrap()); + assert_eq!(duration, Timestamp::new(45_000, scale).unwrap()); + + // A next group overlapping the edge leaves an empty span at the resume point. + let Read::Gap { timestamp, duration } = gap(Duration::from_secs(2), resume) else { + unreachable!() + }; + assert_eq!(timestamp, resume); + assert_eq!(duration, Timestamp::new(0, scale).unwrap()); + } + #[tokio::test] - async fn gaps_mark_the_join_and_holes_while_a_marker_reads_as_an_end() { + async fn gaps_mark_holes_while_a_marker_reads_as_an_end() { let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); let subscription = track.subscribe(moq_net::track::Subscription::default().with_max_delay(Duration::from_secs(10))); @@ -998,10 +1043,11 @@ mod tests { assert_eq!( items(&mut reader).await, [ - "gap", + // Nothing was read before the join, so it is no gap. Legacy carries no duration, so + // the hole starts at the last frame read. "0", "10000", - "gap", + "gap 10000+990000", "1000000", "1010000", "end 1020000", @@ -1021,7 +1067,7 @@ mod tests { write_group(&mut track, 0, &[ts(0)]); write_group(&mut track, 3, &[ts(1_000_000)]); track.finish().unwrap(); - assert_eq!(items(&mut reader).await, ["gap", "0", "gap", "1000000"]); + assert_eq!(items(&mut reader).await, ["0", "gap 0+1000000", "1000000"]); } /// A video group's trailing duration marker is that frame's end, not a declared stop. @@ -1051,7 +1097,7 @@ mod tests { group.finish().unwrap(); } track.finish().unwrap(); - assert_eq!(items(&mut reader).await, ["gap", "0", "33000"]); + assert_eq!(items(&mut reader).await, ["0", "33000"]); } /// An audio terminal opens its group with the endpoint, then the codec's final packets. @@ -1077,7 +1123,7 @@ mod tests { .unwrap(); group.finish().unwrap(); track.finish().unwrap(); - assert_eq!(items(&mut reader).await, ["gap", "0", "20000", "end 50000", "40000"]); + assert_eq!(items(&mut reader).await, ["0", "20000", "end 50000", "40000"]); } #[tokio::test] @@ -1351,7 +1397,7 @@ mod tests { let read = |consumer: &mut Consumer| loop { match consumer.poll_read(&waiter) { Poll::Ready(Ok(Some(Read::Frame { frame, .. }))) => break Some(frame.timestamp), - Poll::Ready(Ok(Some(Read::Gap))) => continue, + Poll::Ready(Ok(Some(Read::Gap { .. }))) => continue, Poll::Pending => break None, other => panic!("unexpected read: {other:?}"), } @@ -2001,7 +2047,10 @@ mod tests { .expect("consumer hung on an evicted group / gap") .unwrap() .unwrap(); - assert!(matches!(next, Read::Gap), "the evicted media is a gap"); + assert!( + matches!(next, Read::Gap { timestamp, duration } if timestamp == ts(0) && duration == ts(150_000)), + "the evicted media is a gap from the last frame to the live group: {next:?}" + ); let next = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(next.timestamp, ts(150_000), "skipped the evicted gap to the live group"); } diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index 26602015cdc..f033ec5622f 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -55,10 +55,18 @@ pub enum Read { group: u64, frame: Frame, }, - /// Media before the next item will never reach this reader: it joined mid-track, or a - /// group was skipped, evicted, or never arrived. A decoder resumes from here without - /// the references or state that media built. - Gap, + /// Media after an item already read will never reach this reader: a group was skipped, + /// evicted, or never arrived. A decoder resumes from here without the references or + /// state that media built. Joining mid-track is not a gap: nothing was read before it. + /// + /// The span is an upper bound, in the scale of the item that follows. + Gap { + /// The furthest presentation point already read. Without a per-frame duration this + /// is the last frame's timestamp, one frame early. + timestamp: moq_net::Timestamp, + /// How long until the next item resumes, so it also covers any declared pause. + duration: moq_net::Timestamp, + }, /// The publisher declared that its media stops at `end`: it paused, switched source, or /// finished. Frames that follow from the same `group` are the stopped encoder's final /// packets (an audio codec's drain), whose output at or past `end` is padding. The next diff --git a/rs/moq-rtc/src/codec/mod.rs b/rs/moq-rtc/src/codec/mod.rs index 20d24b7a712..875b942defb 100644 --- a/rs/moq-rtc/src/codec/mod.rs +++ b/rs/moq-rtc/src/codec/mod.rs @@ -233,7 +233,7 @@ impl Track { let frame = match self.consumer.read().await? { Some(moq_mux::container::Read::Frame { frame, .. }) => frame, // RTP sequence numbers and timestamps already show the hole to a receiver. - Some(moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. }) => continue, + Some(moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. }) => continue, None => return Ok(None), }; let payload = match &self.convert { diff --git a/rs/moq-srt/src/ts.rs b/rs/moq-srt/src/ts.rs index 2766366623c..c1f38d69d29 100644 --- a/rs/moq-srt/src/ts.rs +++ b/rs/moq-srt/src/ts.rs @@ -711,7 +711,7 @@ mod tests { loop { match reader.read().await.unwrap()? { moq_mux::container::Read::Frame { frame, .. } => return Some(frame), - moq_mux::container::Read::Gap | moq_mux::container::Read::End { .. } => continue, + moq_mux::container::Read::Gap { .. } | moq_mux::container::Read::End { .. } => continue, } } } diff --git a/rs/moq-video/src/decode/consumer.rs b/rs/moq-video/src/decode/consumer.rs index 58f1bdf0604..f47c1443d41 100644 --- a/rs/moq-video/src/decode/consumer.rs +++ b/rs/moq-video/src/decode/consumer.rs @@ -168,7 +168,7 @@ impl Consumer { Some(Read::Frame { frame, .. }) => Some(frame), // The next group starts on a keyframe with parameter sets, so neither a // gap nor a declared end flushes the decoder. - Some(Read::Gap | Read::End { .. }) => continue, + Some(Read::Gap { .. } | Read::End { .. }) => continue, None => None, };