diff --git a/quest/m1/export-delay-wire.md b/quest/m1/export-delay-wire.md deleted file mode 100644 index 29f24c7c3c3..00000000000 --- a/quest/m1/export-delay-wire.md +++ /dev/null @@ -1,66 +0,0 @@ -# [S] A fixed-delay export states its whole delay to the relay - -## Goal - -A subscriber that plays a fixed delay D, such as `moq export ts --delay D`, -subscribes with `Delay { min: D, max: D }`, so the relay resets a group with -`Old` only once it is D old, not D/2. A downstream stall the transport -recovers from well within D costs the export no content to relay resets. - -## Plan - -Decided 2026-11-17 (maintainer, `/quest-import` of -[moq-dev/moq#9018](https://github.com/moq-dev/moq/issues/9018)): - -- ✅ A client whose jitter buffer holds at most D sends `max = D`, and a - fixed delay D also sends `min = D` (rejected: keep sending the export's - local skip budget, D/2, as the wire max). The point of the wire max is - that the relay resets with `Old` instead of sending a group once it is D - old; sending D/2 makes it give up at half the budget. -- ✅ The relay is unchanged (rejected: also stop resetting lite-06 streams - that are fully written and only awaiting ACKs, - [moq-dev/moq#4534](https://github.com/moq-dev/moq/pull/4534)). With D on - the wire, the relay's rule (expire once D has passed since the next group - reached it) fires about when the group's last frame misses the - subscriber's own deadline anyway. -- Out of scope: how fast QUIC recovers and the order it drains a backlog in, - owned by [export-order](/quest/m3/export-order.md), - [quic-path-return](/quest/m3/quic-path-return.md), and - [quic-deadline](/quest/m4/quic-deadline.md). - -Facts: - -- `ts::Export::budget()` is `program.delay / 2` - (`rs/moq-mux/src/container/ts/export.rs`), and every track subscription - the export makes uses it as `Delay::max`, with `min` left at 0. - `ts::Release` (the `--passthrough` export) does the same at - `rs/moq-mux/src/container/ts/release.rs:204`. The halving came from - [moq-dev/moq#4645](https://github.com/moq-dev/moq/pull/4645), where it - was a local rule: skip a stalled group once it falls D/2 behind the newest - content, so the group after it has the other half to arrive in. -- In `rs/moq-cli/src/subscribe.rs`, some exports already send - `min = max = D` (`:449`, `:612`) and others send only `max` (`:366`, - `:387`, `:467`, `:482`). Audit every fixed-delay subscriber in moq-cli and - moq-mux the same way, `moq play` included. -- The reporter measured with lite-06, which carries no min on the wire; - lite-07 carries it as Subscriber Min Age. - -Recommendation: the export's local stalled-group skip is the export's own -rule. Keep it only if it still loses no content within D once the wire says -D, judged by mocked-time tests of a stall at a fraction of D. The CLI help -for `--delay` ("skipped once it falls half this behind") changes with it. -Ask the reporter to re-run the blackout sweep from #9018 on the branch. - -Public API: none. Wire: none (different values in existing fields). - -Caused by, for the fix PR's `## Caused by`: - -- [moq-dev/moq#4645](https://github.com/moq-dev/moq/pull/4645) - sent the TS export's local skip budget, D/2, as the subscription's max delay - -## Closes - -- [#9018](https://github.com/moq-dev/moq/issues/9018) - the relay resets a `--delay 2s` export's groups after a 0.6 s stall - -## Related - -- [Export order](/quest/m3/export-order.md) - the drain after an outage, the rest of the loss in #8559 diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 80e24e8f2d9..bbe2960e929 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -1309,7 +1309,7 @@ pub struct BufferedFragmented { pub struct Transport { /// How far behind the source the output runs (e.g. `2s`), like an SRT receiver's latency, /// including how far ahead of its decode time each frame is sent. A frame that arrives - /// later than this is dropped, and a stalled group is skipped once it falls half this + /// later than this is dropped, and a stalled group is skipped once it falls this far /// behind the newest content. #[usage(long, default = "1s")] pub delay: moq_tokio::cli::Duration, diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 4cc5635cb4b..d7005a38f54 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -1414,15 +1414,11 @@ impl Consumer { /// a missing group is waited for from now on. A min past the max starts at the max, since the /// publisher serves nothing older. pub fn set_delay(&mut self, delay: Delay) { + self.set_skip(delay); // An ended reader has no subscription left to tune. let (Reader::Reading(track) | Reader::Failed(track, _)) = &mut self.reader else { return; }; - self.min_delay = delay.min; - self.max_delay = delay - .max - .unwrap_or(std::time::Duration::MAX) - .min(track.info().max_age.unwrap_or(std::time::Duration::MAX)); // The transport enforces the same budget on the subscription itself, so a // tolerance set here has to reach it: otherwise moq-net skips the very groups // this consumer was told to wait for, before they ever get here. It keeps the @@ -1434,6 +1430,25 @@ impl Consumer { } } + /// Skip at `delay`, clamped like [`set_delay`](Self::set_delay), leaving the subscription's + /// delay as it is. A delay past the subscription's skips nothing later, since the transport + /// drops a group first. + pub(crate) fn skipping_at(mut self, delay: Delay) -> Self { + self.set_skip(delay); + self + } + + // Wait and skip at `delay`, its max clamped to the track's max age. + fn set_skip(&mut self, delay: Delay) { + if let Reader::Reading(track) | Reader::Failed(track, _) = &self.reader { + self.min_delay = delay.min; + self.max_delay = delay + .max + .unwrap_or(std::time::Duration::MAX) + .min(track.info().max_age.unwrap_or(std::time::Duration::MAX)); + } + } + /// Skip a missing group once `playhead`, the playout clock this reader feeds, reaches where the /// first group past it begins (default: no clock), rather than waiting out the whole /// [`Delay::min`]. Suits media where a late item is worthless once the next is due, such as diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index dc191baafeb..e54fd82cb0a 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -161,6 +161,8 @@ pub(crate) struct ExportSource { state: SourceState, /// Where the subscription starts behind the live edge, and how late a group may fall. delay: moq_net::track::Delay, + /// The delay the consumer skips and starts at, if not the one the subscription states. + skip: Option, transform: Option, /// Resolved codec configuration record (avcC / hvcC / AudioSpecificConfig / /// OpusHead). Some once the codec config is available - from the catalog @@ -321,6 +323,7 @@ impl ExportSource { state: target.request(source, media), target, delay, + skip: None, transform: None, description: None, video_codec: None, @@ -396,6 +399,14 @@ impl ExportSource { self } + /// Skip and start at `delay` instead of the subscription's own, which still tells the + /// publisher how long to keep a group: for a reader that asks the publisher for more than + /// it plays out itself. + pub(crate) fn skipping_at(mut self, delay: moq_net::track::Delay) -> Self { + self.skip = Some(delay); + self + } + /// The subscription options a builder sets. Each builder consumes a source no poll has /// moved out of [`SourceState::Requesting`], since only the constructors hand one out. fn start(&mut self) -> &mut Start { @@ -622,6 +633,10 @@ impl ExportSource { None => track, }; let consumer = Consumer::new(track, media).continuing(reached); + let consumer = match self.skip { + Some(delay) => consumer.skipping_at(delay), + None => consumer, + }; let consumer = match live { true => consumer.live(), false => consumer, diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index d92e496d6a2..d2d44705738 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -1640,10 +1640,12 @@ where /// arrival skew between them. A frame that arrives after its deadline is dropped, and a /// video track that dropped one resumes at its next keyframe. A source whose clock runs /// further off ours than 13818-1 lets the PCR follow fails the export once it has used half - /// the delay, where a brief network stall only drops what it made late. A stalled group is - /// skipped once it falls half the delay behind the newest content (see - /// [`Consumer`](crate::container::Consumer)), so the group after it still arrives with - /// the other half to spare. + /// the delay, where a brief network stall only drops what it made late. Every track is + /// subscribed with the whole delay as its min and max, so a relay resets a group only once it + /// is that old. The export skips a stalled group once it falls the whole delay behind the + /// newest content (see [`Consumer`](crate::container::Consumer)), so a stall shorter than the + /// delay loses nothing, while a group whose tail never arrives makes the start of the next one + /// late. /// /// With a multiplex rate, a frame that cannot arrive by its decode time fails the export, /// as does video whose HRD declares a buffer delay past the delay. @@ -2576,17 +2578,14 @@ where .filter(|track| track.source.reads(&self.program.source, name, config)); let mut track = match reused { Some(track) => track, - None => match ExportSource::for_video( - &self.program.source, - name, - config, - moq_net::track::Delay::default().with_min(None).with_max(self.budget()), - )? - .map(|s| at_edge(s, self.program.replay)) - { - Some(source) => Track::new(source, kind.clone()), - None => continue, - }, + None => { + match ExportSource::for_video(&self.program.source, name, config, subscribed(self.program.delay))? + .map(|s| at_edge(skipping(s, self.program.delay), self.program.replay)) + { + Some(source) => Track::new(source, kind.clone()), + None => continue, + } + } }; track.kind = kind; track.timing.configure(config, name)?; @@ -2602,17 +2601,14 @@ where .filter(|track| track.source.reads(&self.program.source, name, config)); let mut track = match reused { Some(track) => track, - None => match ExportSource::for_audio( - &self.program.source, - name, - config, - moq_net::track::Delay::default().with_min(None).with_max(self.budget()), - )? - .map(|s| at_edge(s, self.program.replay)) - { - Some(source) => Track::new(source, kind.clone()), - None => continue, - }, + None => { + match ExportSource::for_audio(&self.program.source, name, config, subscribed(self.program.delay))? + .map(|s| at_edge(skipping(s, self.program.delay), self.program.replay)) + { + Some(source) => Track::new(source, kind.clone()), + None => continue, + } + } }; track.buffer = Some(audio_buffer(config, &kind)); track.kind = kind; @@ -2635,11 +2631,10 @@ where Some(track) => track, None => Track::new( stream_edge( - ExportSource::for_stream( - &self.program.source, - name, - moq_net::track::Delay::default().with_min(None).with_max(self.budget()), - )?, + skipping( + ExportSource::for_stream(&self.program.source, name, subscribed(self.program.delay))?, + self.program.delay, + ), &kind, self.program.replay, ), @@ -2681,38 +2676,24 @@ where /// old one stopped, so its first frame counts as a skip: late frames drop, and a leap /// ahead opens a new generation. fn resubscribe(&mut self, catalog: &Catalog, mpegts: &catalog::Mpegts) -> Result<()> { - let budget = self.budget(); + let delay = self.program.delay; for (name, track) in self.program.tracks.iter_mut() { if track.read != Read::Stale { continue; } let source = if let Some(config) = catalog.video.renditions.get(name) { - ExportSource::for_video( - &self.program.source, - name, - config, - moq_net::track::Delay::default().with_min(None).with_max(budget), - )? - .map(|s| at_edge(s, self.program.replay)) + ExportSource::for_video(&self.program.source, name, config, subscribed(delay))? + .map(|s| at_edge(skipping(s, delay), self.program.replay)) } else if let Some(config) = catalog.audio.renditions.get(name) { - ExportSource::for_audio( - &self.program.source, - name, - config, - moq_net::track::Delay::default().with_min(None).with_max(budget), - )? - .map(|s| at_edge(s, self.program.replay)) + ExportSource::for_audio(&self.program.source, name, config, subscribed(delay))? + .map(|s| at_edge(skipping(s, delay), self.program.replay)) } else if mpegts .tracks .get(name) .is_some_and(|t| t.verbatim.is_some() || t.pes_packet.is_some()) { - let source = ExportSource::for_stream( - &self.program.source, - name, - moq_net::track::Delay::default().with_min(None).with_max(budget), - )?; - Some(stream_edge(source, &track.kind, self.program.replay)) + let source = ExportSource::for_stream(&self.program.source, name, subscribed(delay))?; + Some(stream_edge(skipping(source, delay), &track.kind, self.program.replay)) } else { None }; @@ -2739,29 +2720,21 @@ where /// source continues the old one ([`ExportSource::continuing`]), and the frames the old one /// left held for their DTS go out first. fn rebind(&mut self, catalog: &Catalog) -> Result<()> { - let budget = self.budget(); + let delay = self.program.delay; for (name, track) in self.program.tracks.iter_mut().filter(|(_, track)| track.listed) { let (source, kind) = if let Some(config) = catalog.video.renditions.get(name) { if track.source.reads(&self.program.source, name, config) { continue; } - let source = ExportSource::for_video( - &self.program.source, - name, - config, - moq_net::track::Delay::default().with_min(None).with_max(budget), - )?; + let source = ExportSource::for_video(&self.program.source, name, config, subscribed(delay))? + .map(|s| skipping(s, delay)); (source, video_kind(config, name)?) } else if let Some(config) = catalog.audio.renditions.get(name) { if track.source.reads(&self.program.source, name, config) { continue; } - let source = ExportSource::for_audio( - &self.program.source, - name, - config, - moq_net::track::Delay::default().with_min(None).with_max(budget), - )?; + let source = ExportSource::for_audio(&self.program.source, name, config, subscribed(delay))? + .map(|s| skipping(s, delay)); (source, audio_kind(config, name)?) } else { continue; @@ -3969,10 +3942,8 @@ where Ok(()) } - /// How far a stalled group may fall behind the newest content before each source skips it: - /// half the delay. A skip comes once the next group's start is that far behind, and its - /// frames are only read then, so with the whole delay they would arrive at their deadline. - /// Sources state it with no min delay, so a missing group is waited for it too. + /// Half the delay: how long a PES of packed frames may span, and how far a padded output may + /// fall behind before it sheds. fn budget(&self) -> Duration { self.program.delay / 2 } @@ -4575,6 +4546,21 @@ fn video_kind(config: &VideoConfig, name: &str) -> Result { } } +/// The delay a track subscription states for an export running `delay` behind: all of it, as both +/// bounds, so a relay resets a group only once it is that old. +pub(super) fn subscribed(delay: Duration) -> moq_net::track::Delay { + moq_net::track::Delay::default().with_min(delay).with_max(delay) +} + +/// Skip a stalled group once the group after it runs `delay` past it, and wait that long for a +/// missing one, so a stall shorter than the delay loses nothing. The reader delivers in order, so a +/// group whose tail never arrives holds the next one for the whole delay, and the next group's +/// first frames go out late. No min, unlike [`subscribed`], so a live start begins at the newest +/// group rather than reach back the delay. +pub(super) fn skipping(source: ExportSource, delay: Duration) -> ExportSource { + source.skipping_at(moq_net::track::Delay::default().with_min(None).with_max(delay)) +} + /// Start a media track's source at the live edge, the newest group, so the export does not /// begin on a group up to a delay old and carry that lag ever after; unless `replay`. /// Verbatim streams are sparse, and anything of theirs ahead of the first keyframe is diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index f662d5dd779..d8e94b263f7 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -8611,7 +8611,10 @@ async fn jitter_published_after_the_tables_keeps_the_encoders_clock() { } /// Only skipped schedule slots flag the PCR, keeping the program and release clock. -/// Short holes keep the grid continuous; long ones exceed the scheduler's backfill window. +/// Short holes keep the grid continuous; long ones exceed the scheduler's backfill window, and +/// with no delay each flags its resume. With a delay the export waits the whole of it for the +/// missing group, so the group after it is read right at its deadline, and a long hole may be +/// backfilled instead. #[tokio::test(start_paused = true)] async fn a_skipped_group_flags_the_clock() { let cases = [Duration::ZERO, Duration::from_millis(500)] @@ -8670,18 +8673,23 @@ async fn a_skipped_group_flags_the_clock() { out.extend(drain_frames(&mut export).await); assert_eq!(export.discontinuity(), 0, "a skipped group is not a new program clock"); - assert_skip_discontinuity(&out, 2 * usize::from(missing == 60 && delay.is_zero())); + let resumes: Vec = [20 + missing, 40 + 2 * missing] + .into_iter() + .filter(|_| missing == 60) + .map(|sequence| (60_000 + sequence * 100) * 90) + .collect(); + let flags = count_discontinuity(&out); + match delay.is_zero() { + true => assert_eq!(flags, resumes.len(), "each long hole flags its resume"), + false => assert!(flags <= resumes.len(), "{flags} flags for {} long holes", resumes.len()), + } + assert_skip_discontinuity(&out, flags); let ts: Vec = out.iter().flat_map(|f| f.payload.iter().copied()).collect(); for pair in out.windows(2) { let jumped = pair[1].timestamp.as_micros().unwrap() > pair[0].timestamp.as_micros().unwrap() + 25_000; assert_eq!(count_discontinuity(&pair[1..]), usize::from(jumped)); } // Each flag lands right before the audio that resumes after its hole. - let resumes: Vec = [20 + missing, 40 + 2 * missing] - .into_iter() - .filter(|_| missing == 60 && delay.is_zero()) - .map(|sequence| (60_000 + sequence * 100) * 90) - .collect(); let mut reader = TsPacketReader::new(Cursor::new(&ts)); let (mut flagged, mut after) = (false, Vec::new()); while let Some(packet) = reader.read_ts_packet().unwrap() { @@ -8694,7 +8702,11 @@ async fn a_skipped_group_flags_the_clock() { after.extend(pes.header.pts.map(|pts| pts.as_u64())); } } - assert_eq!(after, resumes, "the first PES after each flag resumes its hole"); + assert_eq!(after.len(), flags); + assert!( + after.iter().all(|pts| resumes.contains(pts)), + "the first PES after each flag resumes its hole: {after:?}, holes resume at {resumes:?}" + ); let (_, audio) = collect_pes_pts(&ts); assert!( audio.iter().any(|&pts| pts >= (60_000 + (70 + 2 * missing) * 100) * 90), @@ -10313,6 +10325,26 @@ async fn a_follower_keeps_a_return_awaiting_its_catalog_when_a_stitch_goes() { assert_eq!(start.elapsed(), follow); } +/// Each track subscription states the whole delay as its min and max, so a relay holds a group +/// as long as the export could still play it. +#[tokio::test(start_paused = true)] +async fn a_subscription_states_the_whole_delay() { + let origin = crate::source::produce_origin(); + let (live, mut catalog) = publish_bare(&origin, None); + let mut audio = aac_rendition(&mut live.broadcast().clone(), &mut catalog, "a.aac"); + write_aac(&mut audio, 0); + let delay = Duration::from_secs(2); + let mut export = open_export(crate::source::resolved(&origin, "live").await, delay) + .await + .unwrap(); + let _ = kio::clock::timeout(Duration::from_secs(1), export.next()).await; + assert_eq!( + audio.track().subscription().unwrap().delay, + moq_net::track::Delay::default().with_min(delay).with_max(delay), + ); + drop((live, catalog)); +} + /// A program that ends while stitching, then returns after the unresolved replacement goes, /// resumes at normal priority. #[tokio::test(start_paused = true)] @@ -12661,8 +12693,8 @@ async fn fixed_delay_keeps_buffered_pes_ahead_of_the_clock() { end.unwrap(); all.extend(frames); assert_decodes_after_the_clock(&all); - // Joining retains half the delay plus its boundary frame. - assert_eq!(pes_count(&all), 76); + // Joining retains the whole delay plus its boundary frame. + assert_eq!(pes_count(&all), 151); assert_skip_discontinuity(&all, 0); } diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index bbb00e6678a..0cb24cf5c36 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -293,27 +293,9 @@ impl Live { if stall(tick) { self.stalling = true; } else if !self.stalling { - let slice = if keyframe { - vec![0x65u8; 3_000] - } else { - vec![0x41u8; self.delta_size] - }; - let presented = match self.reorder { - true => (tick - tick % GOP + PYRAMID[(tick % GOP) as usize] + PYRAMID_DELAY) * VIDEO_US, - false => video, - }; + let frame = self.video_frame(tick); let group = self.group.as_mut().expect("the first tick is a keyframe"); - HangContainer::Legacy(crate::container::Kind::Video) - .write( - group, - &[Frame { - timestamp: Timestamp::from_micros(presented.strict_add_signed(self.offset)).unwrap(), - duration: None, - payload: length_prefixed(&[&slice]), - keyframe, - }], - ) - .unwrap(); + write_video(group, frame); } self.tick += 1; } @@ -323,6 +305,87 @@ impl Live { } } + /// The video frame of `tick`. + fn video_frame(&self, tick: u64) -> Frame { + let keyframe = tick.is_multiple_of(GOP); + let slice = if keyframe { + vec![0x65u8; 3_000] + } else { + vec![0x41u8; self.delta_size] + }; + let presented = match self.reorder { + true => (tick - tick % GOP + PYRAMID[(tick % GOP) as usize] + PYRAMID_DELAY) * VIDEO_US, + false => tick * VIDEO_US, + }; + Frame { + timestamp: Timestamp::from_micros(presented.strict_add_signed(self.offset)).unwrap(), + duration: None, + payload: length_prefixed(&[&slice]), + keyframe, + } + } + + /// Stall the whole path for `stall` from the next video tick, which must lie in the open group + /// with a group boundary within the stall: nothing arrives, and then all that was sent + /// meanwhile lands at once, newest group first, as a relay drains a backlog in descending + /// group order. The legs read the audio and the newer group before the older group's tail + /// lands. + fn stall_path(&mut self, stall: Duration, legs: &mut [&mut Leg]) { + let (video_at, _) = self.send_times(self.tick * VIDEO_US, 0); + let resume = video_at.saturating_add(stall); + let (mut tail, mut newer) = (Vec::new(), Vec::new()); + let mut boundary = None; + let mut audio = Vec::new(); + loop { + let (video_at, audio_at) = self.send_times(self.tick * VIDEO_US, self.audio_index * AUDIO_US); + if video_at.min(audio_at) >= resume { + break; + } + if audio_at < video_at { + audio.push(self.audio_index); + self.audio_index += 1; + } else { + let frame = self.video_frame(self.tick); + if frame.keyframe { + assert!(boundary.is_none(), "the stall crosses one group boundary"); + boundary = Some(self.tick); + } + match boundary { + None => tail.push(frame), + Some(_) => newer.push(frame), + } + self.tick += 1; + } + } + let boundary = boundary.expect("the stall crosses one group boundary"); + assert!(!tail.is_empty(), "the stall starts within a group"); + self.advance(resume, legs); + + let end = std::mem::take(&mut self.audio_index); + for index in audio { + self.audio_index = index; + self.send_audio(); + } + self.audio_index = end; + let start = (boundary * VIDEO_US).strict_add_signed(self.offset); + let mut group = self.video.append_group(Timestamp::from_micros(start).unwrap()).unwrap(); + for frame in newer { + write_video(&mut group, frame); + } + for leg in legs.iter_mut() { + leg.poll_now(); + } + + let mut older = self.group.replace(group).expect("the stall starts in an open group"); + for frame in tail { + write_video(&mut older, frame); + } + older.finish().unwrap(); + for leg in legs.iter_mut() { + leg.poll_now(); + } + } + /// Write the next audio frame. fn send_audio(&mut self) { let index = self.audio_index; @@ -416,6 +479,13 @@ impl Live { } } +/// Write one video frame to `group`. +fn write_video(group: &mut moq_net::group::Producer, frame: Frame) { + HangContainer::Legacy(crate::container::Kind::Video) + .write(group, &[frame]) + .unwrap(); +} + /// One receiver: an exporter and what it rendered, with the instant each slot went out. struct Leg { export: Export, @@ -698,22 +768,30 @@ const LAGS: [i64; 2] = [300_000, -300_000]; const CLOCK_SLACK: f64 = 2.0 * SLOT_MS; /// Join mid-group with `lag`, render 7.2 s with the video ticks `stall` accepts stalled, and -/// return the leg's units. -async fn join(lag: i64, stall: impl Fn(u64) -> bool) -> Vec { +/// return the leg's units and how many frames it dropped as late. +async fn join(lag: i64, stall: impl Fn(u64) -> bool) -> (Vec, u64) { let mut live = Live::new(1.0, lag).await; live.run(JOIN, &mut [], |_| false); let mut leg = live.join(Duration::ZERO).await; live.run(24 * GOP, &mut [&mut leg], stall); - assert_eq!(leg.export.dropped(), 0, "every frame arrived inside its deadline"); - leg.units() + (leg.units(), leg.export.dropped()) } -/// Assert a skip left the video where it is without one, against the audio, from 4 s after the -/// leg's first video unit. +/// Assert a skip cost at most the group after the stalled one and left the video where it is +/// without one, against the audio, from 4 s after the leg's first video unit. +/// +/// The stalled group's tail never arrives, so the export waits the whole delay for it before it +/// skips to the next group, whose first frames are then due: they go late, and the video resumes +/// at the keyframe after them. The wait is bounded by the delay, so no more than that group goes. async fn assert_one_clock(stall: impl Fn(u64) -> bool + Copy) { let mut moved = Vec::new(); for lag in LAGS { - let (clean, skipped) = (join(lag, |_| false).await, join(lag, stall).await); + let ((clean, none), (skipped, dropped)) = (join(lag, |_| false).await, join(lag, stall).await); + assert_eq!(none, 0, "every frame of a clean source arrived inside its deadline"); + assert!( + dropped <= GOP, + "lag {lag}: {dropped} frames went late, more than the group the stall held" + ); let from = after_first_video(&clean, 4_000_000); let (want, got) = (video_behind_audio(&clean, from), video_behind_audio(&skipped, from)); if (got - want).abs() > CLOCK_SLACK { @@ -749,6 +827,45 @@ async fn a_skip_while_running_keeps_one_clock() { assert_one_clock(|tick| (6 * GOP + 7..7 * GOP).contains(&tick)).await; } +/// A stall of the whole path longer than half the delay but shorter than it, across a group +/// boundary, loses nothing, though the newer group lands before the older group's tail: the +/// export waits the whole delay for the tail rather than skip it. +#[tokio::test(start_paused = true)] +async fn a_path_stall_within_the_delay_loses_nothing() { + let mut lost = Vec::new(); + for stall in [300, 400, 475].map(Duration::from_millis) { + for before in [1, 3] { + let mut live = Live::new(1.0, 0).await; + let mut leg = live.join(Duration::ZERO).await; + live.run(6 * GOP - before, &mut [&mut leg], |_| false); + live.stall_path(stall, &mut [&mut leg]); + live.run(12 * GOP, &mut [&mut leg], |_| false); + let ticks: Vec = leg + .units() + .iter() + .filter(|unit| unit.pid == VIDEO_PID) + .map(|unit| unit.decode / at90(VIDEO_US)) + .collect(); + let skipped = ticks.windows(2).filter(|pair| pair[1] != pair[0] + 1).count(); + let dropped = leg.export.dropped(); + if dropped > 0 || skipped > 0 || ticks.last().is_none_or(|last| *last < 7 * GOP) { + lost.push(format!( + "{} ms stall from {before} frames before a boundary: {dropped} dropped, {skipped} skips, \ + video {:?}..{:?}", + stall.as_millis(), + ticks.first(), + ticks.last(), + )); + } + } + } + assert!( + lost.is_empty(), + "a stall within the delay lost content:\n{}", + lost.join("\n") + ); +} + // A TS source sends its video up to a second ahead of its decode time and its audio just in time, // and a passed-through AC-3 PES of nine sync frames arrives only once its 288 ms are complete, so // audio of a given decode time can reach the receiver more than a delay after the video. A clock diff --git a/rs/moq-mux/src/container/ts/release.rs b/rs/moq-mux/src/container/ts/release.rs index 5fd720fc395..8f7c18d0360 100644 --- a/rs/moq-mux/src/container/ts/release.rs +++ b/rs/moq-mux/src/container/ts/release.rs @@ -6,6 +6,7 @@ use std::time::Duration; use kio::clock::Instant; use super::error::unsupported; +use super::export; #[cfg(feature = "async")] use crate::catalog::CatalogFormat; @@ -198,13 +199,8 @@ impl Release { }; match &self.track { None => { - let source = ExportSource::for_stream( - &self.source, - &m2ts.track, - // Half the delay, as the export's budget, and no min, so a missing group is - // waited for it too. - moq_net::track::Delay::default().with_min(None).with_max(self.delay / 2), - )?; + let source = ExportSource::for_stream(&self.source, &m2ts.track, export::subscribed(self.delay))?; + let source = export::skipping(source, self.delay); let source = match self.replay { true => source, false => source.live(), diff --git a/rs/moq-srt/src/server.rs b/rs/moq-srt/src/server.rs index fd8e11e3c38..3b8c77b6c09 100644 --- a/rs/moq-srt/src/server.rs +++ b/rs/moq-srt/src/server.rs @@ -973,9 +973,10 @@ mod tests { .unwrap(); received.push(first); - // The publisher declares a break and continues forward. + // The publisher declares a break and continues forward, with half a second written at + // once: a longer burst would leave the marker's group stale before the egress reads it. producer.discontinuity().unwrap(); - write(&mut producer, 30, OFFSET + 3_000_000); + write(&mut producer, 5, OFFSET + 3_000_000); // Drain until the muxer flags the break, which it does exactly once, on the new // generation's leading clock packet. diff --git a/rs/moq-srt/src/ts.rs b/rs/moq-srt/src/ts.rs index b1f4455e1fe..13f6717dbd4 100644 --- a/rs/moq-srt/src/ts.rs +++ b/rs/moq-srt/src/ts.rs @@ -225,7 +225,7 @@ impl Subscriber { /// /// `options.latency` is the muxer's delay, but no less than [`MIN_DELAY`]: the output runs /// that far behind the source, a frame arriving later is dropped, and a stalled group is - /// skipped after half of it. We reuse the locally configured SRT receive latency for + /// skipped once it falls that far behind the newest content. We reuse the locally configured SRT receive latency for /// it, the same budget an SRT hop gives a packet. It's the configured value, /// not the handshake result (srt-tokio doesn't expose the negotiated latency), /// so a peer that requests a higher receive latency gets a larger actual