From 6c599174c2bfdfd21bd47de1da290ad1c5bc06f6 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 02:55:42 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m1/ts-padded-shed.md Quest-Claim: 193c79f9695436bd From a1caa22276c11c85af790f70a697a1023e1a6dd4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 09:57:09 -0800 Subject: [PATCH 2/3] fix(moq-mux): a padded TS export sheds when its sink falls behind A `--mux-rate` export laid every slot it missed once its sink resumed (a 10 s block wrote 400 slots at once), and a sink slower than the rate left frames unread until the jitter buffer read the export's own lag as a source clock about -10^6 ppm off and failed. Once the jitter buffer's front has been overdue for more than half the delay since a read last caught up, the export now drops what is due by then (each track resuming at its next sync frame, and any frame read later that was due before the cut counted late and kept off the clock's steering) and rewinds, so the grid resumes on the media still to come with a flagged PCR discontinuity. Closes #8383 Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 - quest/m1/ts-padded-shed.md | 46 ------ rs/moq-mux/src/container/ts/export.rs | 42 +++++ rs/moq-mux/src/container/ts/export_test.rs | 6 + .../src/container/ts/export_timing_test.rs | 143 +++++++++++++++++- rs/moq-mux/src/jitter.rs | 76 ++++++++-- 6 files changed, 251 insertions(+), 63 deletions(-) delete mode 100644 quest/m1/ts-padded-shed.md diff --git a/quest/m1/README.md b/quest/m1/README.md index f578c647594..77bb7f8d52c 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -41,7 +41,6 @@ moves what remains of them once their claims merge. - [Buffered playback subscribes in ascending order](/quest/m1/buffered-ascending.md) - a buffered `` asks for the oldest group first, so a voice reply's opening arrives first, and audio's head hold shrinks to the floor - [Lifecycles are explicit state](/quest/m1/explicit-state/README.md) - every lifecycle the 2026-11-07 audit found implicit is one enum or union, fixing the bugs in the 11 lifecycles it redesigns, with review and agent rules against new ones - [Track drop runs are forgotten below the floor](/quest/m1/drop-runs-prune.md) - a track's dropped-group record stays proportional to its window in Rust and JS -- [A padded TS export sheds when its sink falls behind](/quest/m1/ts-padded-shed.md) - a `--mux-rate` export skips stale slots with a flagged PCR jump instead of back-filling them or blaming the source - [A resumed QUIC client gives up on its own idle timer](/quest/m1/quic-resume-idle.md) - a client stopped past its idle timeout closes within about one timeout of resuming, without waiting on a stateless reset - [A reused moq-transport alias delivers a group that beats its SUBSCRIBE_OK](/quest/m1/ietf-alias-reuse.md) - a new SUBSCRIBE clears its track's alias tombstone, so moqxr's alias-0 catalog refetch no longer hangs - [Unused crates, scripts, and files are deleted](/quest/m1/unused-code.md) - moq-room, the moq-native tombstone, demo/throttle, sh/dart/publish.sh, demo/web's Solid deps, and unpublished packages' release notes go diff --git a/quest/m1/ts-padded-shed.md b/quest/m1/ts-padded-shed.md deleted file mode 100644 index d95b990ba6e..00000000000 --- a/quest/m1/ts-padded-shed.md +++ /dev/null @@ -1,46 +0,0 @@ -# [M] A padded TS export sheds when its sink falls behind - -## Goal - -A `moq export ts` with a mux rate (constant bitrate padded with null packets) -never writes a burst of back-filled slots after its sink blocks, and never -blames the source's clock for a sink slower than the mux rate. Like the -unpadded path, once it falls more than the budget (half the delay) behind, it -sheds: it skips the stale slots and resumes the PCR grid at the current time, -flagging the jump as a PCR discontinuity. - -## Plan - -Planned 2026-11-09 (maintainer, /quest-import of -[moq-dev/moq#8383](https://github.com/moq-dev/moq/issues/8383)). Milestone: -✅ m1 (rejected: m3, m4), since it is an additive fix to shipped code. - -The report, verified against the code: the slot grid follows the jitter -clock, and `lay_due` (`rs/moq-mux/src/container/ts/export.rs`) lays every -slot already due in one pass. The CLI awaits `stdout.write_all` between -`next()` calls, so a blocked stdout stops polling and the resume lays out -every missed slot at once (75 MB of nulls after a 60 s block). Under a slow -reader, a non-empty `program.queue` returns before the jitter buffer -releases. The buffer fills, the tracks stop reading, and `steer` -(`rs/moq-mux/src/jitter.rs`) reads the shrinking slack as a source clock -about -10^6 ppm off and exits. - -Decided 2026-11-09 (maintainer): ✅ shed like the unpadded path (rejected: -re-anchor after a block and error on a sustained slow sink; won't fix). A -single mechanism covers both cases: a 60 s block gives one flagged jump, and -a sink slower than the mux rate gives periodic jumps. - -Guidance: skip the stale slots instead of generating them, through the -existing PCR discontinuity flag. Re-anchor the jitter clock along with the -grid, or `steer` still counts the output's own lag against the source. A -re-anchor is a clock jump, so it must set the flag -([moq-dev/moq#7981](https://github.com/moq-dev/moq/pull/7981) flags only -real jumps). Test on mock time: a sink blocked past the budget resumes with -one flagged discontinuity and no backlog, a sink at half the mux rate keeps a -bounded lag, and neither trips the drift error. - -Public API: none expected. Wire: none. - -## Closes - -- [#8383](https://github.com/moq-dev/moq/issues/8383) - a padded export back-fills every missed slot after stdout blocks diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index e623914922e..fe1e9d35aac 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -126,6 +126,8 @@ where active_program: u64, follow_until: Option, follow_timer: Option>>, + /// When a read last found nothing more due: the output had caught up then ([`Self::shed`]). + caught_up: Option, } /// Input and mux state belonging to one publisher instance. The release clock @@ -1661,6 +1663,7 @@ where active_program: 0, follow_until: None, follow_timer: None, + caught_up: None, } } @@ -1704,6 +1707,14 @@ where /// muxer has no use for it. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll>> { + let poll = self.poll_frame(waiter); + if poll.is_pending() { + self.caught_up = Some(kio::clock::now()); + } + poll + } + + fn poll_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { if self.pending.is_some() { self.stop_old(waiter)?; } @@ -1799,6 +1810,9 @@ where // 4. Mux each frame the jitter buffer lets go (the first carries the buffered // PAT/PMT), then lay out every grid slot whose time has come ([`Self::lay_due`]). + if self.program.queue.is_empty() { + self.shed(); + } loop { if let Some(out) = self.pop() { return Poll::Ready(Ok(Some(out))); @@ -3887,6 +3901,34 @@ where self.program.delay / 2 } + /// Shed what a padded output fell more than [`Self::budget`] behind on, as when its sink + /// blocks or drains slower than the multiplex rate. Every slot it missed would otherwise go + /// out at once, and the frames it left unread would reach the jitter buffer as late as a + /// source clock running far slow. Instead it drops what is due by now and resumes the grid + /// on the media still to come, flagging the jump as a PCR discontinuity. + /// + /// It is behind by how long the jitter buffer's front has been overdue since a read last + /// caught up: a frame held back while the output keeps up (for the program's tables, or a + /// generation still going out) is no lag of the sink's, and a source that fell silent leaves + /// nothing overdue. + fn shed(&mut self) { + // Once every source has ended, the tail goes out late if it must. + let live = self.program.tracks.values().any(|track| track.read == Read::Live); + if self.program.mux_rate.is_none() || self.held.is_some() || !live { + return; + } + let (Some(due), Some(caught_up)) = (self.jitter.front_deadline(), self.caught_up) else { + return; + }; + let behind = kio::clock::now().saturating_duration_since(due.max(caught_up)); + if behind <= self.budget() { + return; + } + tracing::warn!(?behind, "MPEG-TS output fell behind its multiplex rate, shedding"); + self.jitter.shed(kio::clock::now()); + self.rewind(); + } + /// Lay out every grid slot whose time has come, and wake when the next one is due. fn lay_due(&mut self, waiter: &kio::Waiter) -> anyhow::Result<()> { while let Some((known, at)) = self.slot_due() { diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 5689c40e792..417a036fcc3 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -8213,6 +8213,12 @@ async fn export_reordered( } rig.finish(); let mut b = b.unwrap(); + // Side by side: one left unread while the other drains would be a blocked sink. + while let Some(deadline) = [a.next_due(), b.next_due()].into_iter().flatten().min() { + kio::clock::sleep_until(deadline).await; + out_a.extend(poll_frames(&mut a)); + out_b.extend(poll_frames(&mut b)); + } out_a.extend(drain_frames(&mut a).await); out_b.extend(drain_frames(&mut b).await); (out_a, out_b) diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index 05054a88923..45c6fd6aa8c 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -255,6 +255,8 @@ impl Live { waiter: kio::Waiter::new(Waker::from(woken.clone())), woken, out: Vec::new(), + sink: None, + busy: None, } } @@ -347,12 +349,30 @@ impl Live { } } - /// End the broadcast and let every leg render what it holds. + /// End the broadcast and let every leg render what it holds, side by side: a leg left + /// unread while another drains would be a blocked sink. fn finish(&mut self, legs: &mut [&mut Leg]) { self.finish_media(); self.catalog.finish().unwrap(); - for leg in legs.iter_mut() { - leg.drain(&self.clock); + let mut ended = vec![false; legs.len()]; + loop { + for (leg, ended) in legs.iter_mut().zip(&mut ended) { + *ended = *ended || leg.poll(); + } + let Some(deadline) = legs + .iter() + .zip(&ended) + .filter(|(_, ended)| !**ended) + .map(|(leg, _)| { + leg.export + .next_due() + .expect("export parked without ending or scheduled output") + }) + .min() + else { + break; + }; + self.clock.advance(deadline.max(self.clock.now())); } } @@ -407,6 +427,11 @@ struct Leg { waiter: kio::Waiter, woken: Arc, out: Vec<(Instant, Frame)>, + /// The rate its output drains at, in bits per second, if it drains slower than it is read: + /// each frame keeps it from reading again until the frame is written. + sink: Option, + /// It reads nothing until this instant, as a sink still writing or a blocked stdout. + busy: Option, } struct Woken(AtomicBool); @@ -424,6 +449,9 @@ impl Wake for Woken { impl Leg { /// When this leg next reads on its own. fn wake(&self) -> Option { + if self.busy.is_some() { + return self.busy; + } match self.every.is_zero() { true => self.export.next_due(), false => Some(self.next_poll), @@ -433,18 +461,29 @@ impl Leg { /// Read if anything woke the export since it last read, as the task polling it would: a read /// nothing woke finds nothing new. fn poll_now(&mut self) { - if self.every.is_zero() && self.woken.0.load(Ordering::Relaxed) { + if self.busy.is_none() && self.every.is_zero() && self.woken.0.load(Ordering::Relaxed) { assert!(!self.poll(), "export ended while the broadcast is live"); } } /// Read everything ready now, returning whether the broadcast ended. fn poll(&mut self) -> bool { - self.woken.0.store(false, Ordering::Relaxed); let now = kio::clock::now(); + if self.busy.is_some_and(|busy| busy > now) { + return false; + } + self.busy = None; + self.woken.0.store(false, Ordering::Relaxed); while let std::task::Poll::Ready(frame) = self.export.poll_next(&self.waiter) { match frame.expect("exporter error") { - Some(frame) => self.out.push((now, frame)), + Some(frame) => { + let bits = frame.payload.len() as u64 * 8; + self.out.push((now, frame)); + if let Some(sink) = self.sink { + self.busy = Some(now.saturating_add(Duration::from_secs_f64(bits as f64 / sink as f64))); + return false; + } + } None => return true, } } @@ -1102,6 +1141,98 @@ async fn a_source_far_off_real_time_fails_within_seconds() { live.run(10_000_000 / VIDEO_US, &mut [&mut leg], |_| false); } +// A padded output writes every slot of its grid, so a sink that stops reading, or reads slower +// than the multiplex rate, would otherwise face every slot it missed at once, and the backlog's +// own wait would read to the jitter buffer as a source clock running far slow. + +/// How far a padded export may fall behind before it sheds: half the delay. +const BUDGET: Duration = Duration::from_millis(DELAY.as_millis() as u64 / 2); + +/// How long after the source's clock each slot written at or after `from` went out. +fn holds(live: &Live, leg: &Leg, from: Instant) -> Vec { + leg.out + .iter() + .filter(|(sent, _)| *sent >= from) + .map(|(sent, frame)| sent.saturating_duration_since(live.sent(frame.timestamp.as_micros() as u64))) + .collect() +} + +/// How many PCRs flag a discontinuity in the slots written at or after `from`. +fn flagged(leg: &Leg, from: Instant) -> usize { + leg.out + .iter() + .filter(|(sent, _)| *sent >= from) + .flat_map(|(_, frame)| frame.payload.chunks(188).map(<[u8]>::to_vec).collect::>()) + .filter(|packet| packet[3] & 0x20 != 0 && packet[4] > 0 && packet[5] & 0x10 != 0 && packet[5] & 0x80 != 0) + .count() +} + +/// A sink that blocks for ten seconds resumes on the current slot, the jump flagged once, rather +/// than being handed every slot it missed. +#[tokio::test(start_paused = true)] +async fn a_blocked_sink_resumes_on_the_current_slot() { + let mut live = Live::new(1.0, 0).await; + let mut leg = live.join(Duration::ZERO).await; + live.run(10 * GOP, &mut [&mut leg], |_| false); + let normal = *holds(&live, &leg, live.start).iter().max().unwrap(); + + let resume = kio::clock::now().saturating_add(Duration::from_secs(10)); + leg.busy = Some(resume); + live.run(10 * GOP + 10_000_000 / VIDEO_US + 10 * GOP, &mut [&mut leg], |_| false); + + let burst = leg.out.iter().filter(|(sent, _)| *sent == resume).count(); + let slots = (BUDGET.as_millis() / PCR_INTERVAL.as_millis()) as usize + 2; + assert!( + burst <= slots, + "{burst} slots went out on resuming, past the {slots} the budget holds" + ); + assert_eq!(flagged(&leg, resume), 1, "the jump is flagged once"); + let worst = *holds(&live, &leg, resume.saturating_add(BUDGET)).iter().max().unwrap(); + assert!( + worst <= normal + PCR_INTERVAL, + "held {worst:?} after resuming, {normal:?} before" + ); + assert!( + leg.units() + .iter() + .any(|unit| unit.sent > resume && unit.pid == VIDEO_PID), + "video resumed" + ); +} + +/// A sink draining at half the multiplex rate keeps a bounded lag, shedding what it cannot +/// write, and never reads to the jitter buffer as a slow source clock. The export lays at most +/// the budget's worth of missed slots at once, which this sink takes twice as long to write, so +/// it falls up to a delay behind. +#[tokio::test(start_paused = true)] +async fn a_slow_sink_sheds_and_keeps_its_lag() { + let mut live = Live::new(1.0, 0).await; + let mut leg = live.join(Duration::ZERO).await; + live.run(10 * GOP, &mut [&mut leg], |_| false); + let normal = *holds(&live, &leg, live.start).iter().max().unwrap(); + + let slow = kio::clock::now(); + leg.sink = Some(MUX_RATE / 2); + live.run(10 * GOP + 120_000_000 / VIDEO_US, &mut [&mut leg], |_| false); + + let worst = *holds(&live, &leg, slow).iter().max().unwrap(); + assert!( + worst <= normal + DELAY, + "held {worst:?} at half the rate, {normal:?} before" + ); + let video = leg + .units() + .iter() + .filter(|unit| unit.sent > slow && unit.pid == VIDEO_PID) + .count(); + assert!(video > 0, "no video went out"); + assert_eq!( + leg.export.stats().release.out_of_tolerance, + 0, + "the sink read as source drift" + ); +} + // A 1+1 pair is two receivers of one broadcast whose outputs a seamless switch compares packet // for packet. They join at different times and read with different cadences, and from the moment // they overlap they must render the same bytes. The continuity counters are the known exception diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index ab66b4ba5b8..feb374e8baa 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -161,6 +161,12 @@ impl Buffer { { self.writing(key).lose(key, ahead); } + pub fn shed(&mut self, before: Instant) { + if let Some(Handoff::Committed(committed)) = self.handoff.as_deref_mut() { + committed.old.shed(before); + } + self.active.shed(before); + } pub fn discard(&mut self, ready: &Ready) { if let Some(Handoff::Committed(committed)) = self.handoff.as_deref_mut() && !committed.keys.contains(&ready.track) @@ -241,9 +247,14 @@ impl Buffer { for (_, ready) in held { let track = engine.tracks.get_mut(&ready.track).expect("the previewed track"); // The issued start is owned by the caller even while it waits on the old tail. - track - .queue - .push_front((ready.generation, ready.decode, ready.deadline, ready.item, ready.bytes)); + track.queue.push_front(( + ready.generation, + ready.decode, + ready.deadline, + ready.item, + ready.bytes, + ready.sync, + )); track.bytes += ready.bytes; } let first = engine @@ -421,6 +432,7 @@ pub(crate) struct Ready { /// Whether it is the first frame of its track to go out. pub first: bool, bytes: usize, + sync: bool, } /// What [`Buffer::push`] did with a frame. @@ -533,6 +545,9 @@ pub(crate) struct Engine { expect: BTreeSet, /// The latest deadline of a frame gone out. released: Option, + /// Everything due before it was shed ([`Self::shed`]), so a frame due before it is late however + /// early it reached us. + shed: Option, /// What the clock is steered on, since it last started. steer: Steer, /// How much more slack each track had than the track sent latest, in seconds, in the last @@ -756,8 +771,9 @@ enum Admit { } struct Track { - /// Frames in arrival order, each with its generation, decode time, deadline and bytes. - queue: VecDeque<(u64, Decode, Instant, T, usize)>, + /// Frames in arrival order, each with its generation, decode time, deadline, bytes, and + /// whether it is a sync frame. + queue: VecDeque<(u64, Decode, Instant, T, usize, bool)>, /// Wake the reader when its future tail enters the admission window. read_timer: Option>>, /// What its queued frames hold against the byte budget. @@ -786,6 +802,7 @@ impl Engine { horizon: None, expect: BTreeSet::new(), released: None, + shed: None, steer: Steer::default(), leads: BTreeMap::new(), timer: None, @@ -864,7 +881,8 @@ impl Engine { let generation = track.generation; return Ok(self.queue(key, arrival, generation, None, false)); } - if self.released.is_some() && !self.delay.is_zero() { + let shed = deadline.zip(self.shed).is_some_and(|(deadline, shed)| deadline < shed); + if self.released.is_some() && !self.delay.is_zero() && !shed { self.steer(&key, deadline, &arrival)?; } Ok(self.queue(key, arrival, clock.generation, deadline, false)) @@ -1059,7 +1077,10 @@ impl Engine { } _ if track.admit == Admit::Waiting && !arrival.sync => Push::Waiting, _ if track.admit == Admit::Resumed && arrival.leading => Push::Waiting, - Some(deadline) if arrival.arrived <= deadline || self.delay.is_zero() => { + Some(deadline) + if (arrival.arrived <= deadline && self.shed.is_none_or(|shed| deadline >= shed)) + || self.delay.is_zero() => + { // A frame past the checks above while waiting is a sync frame. if arrival.sync { track.admit = match track.admit { @@ -1071,7 +1092,7 @@ impl Engine { track.bytes += cost; track .queue - .push_back((generation, arrival.decode, deadline, arrival.item, cost)); + .push_back((generation, arrival.decode, deadline, arrival.item, cost, arrival.sync)); self.horizon = self.horizon.max(Some(deadline)); Push::Queued } @@ -1376,7 +1397,8 @@ impl Engine { && (self.delay.is_zero() || now >= *deadline) { let track = self.tracks.get_mut(key).expect("the front's track"); - let (generation, decode, deadline, item, bytes) = track.queue.pop_front().expect("the front frame"); + let (generation, decode, deadline, item, bytes, sync) = + track.queue.pop_front().expect("the front frame"); track.bytes -= bytes; let first = !std::mem::replace(&mut track.started, true); self.released = self.released.max(Some(deadline)); @@ -1392,6 +1414,7 @@ impl Engine { deadline, first, bytes, + sync, }); } let Some(wake) = front.map(|(deadline, _)| deadline).into_iter().chain(acquired).min() else { @@ -1435,12 +1458,40 @@ impl Engine { } } + /// Drop every frame due before `before`, which an output that fell that far behind can no + /// longer send in time, and any read later that is. A track that loses a frame resumes at its + /// next sync frame. + pub fn shed(&mut self, before: Instant) { + self.shed = self.shed.max(Some(before)); + for track in self.tracks.values_mut() { + let mut lost = 0; + while let Some((_, _, deadline, _, bytes, sync)) = track.queue.front() + && (*deadline < before || (lost > 0 && !sync)) + { + track.bytes -= bytes; + track.queue.pop_front(); + lost += 1; + } + if lost == 0 { + continue; + } + match track.started { + true => self.dropped += lost, + false => self.trimmed += lost, + } + track.admit = match track.queue.is_empty() { + true => Admit::Waiting, + false => Admit::Resumed, + }; + } + } + /// The frame `key` releases next, if it has one queued. pub fn peek(&self, key: &Q) -> Option<&T> where K: Borrow, { - let (_, _, _, item, _) = self.tracks.get(key)?.queue.front()?; + let (_, _, _, item, ..) = self.tracks.get(key)?.queue.front()?; Some(item) } @@ -1469,6 +1520,11 @@ impl Engine { self.steer.tracks.remove(key); } + /// When the frame that goes out next is due, if one is queued. + pub fn front_deadline(&self) -> Option { + self.front_available(None).map(|(deadline, _)| deadline) + } + /// When the next queued frame is due, or the acquisition under way ends. #[cfg(test)] pub fn next_deadline(&self) -> Option { From c52009b81c90eb43389105d032ecd97d2736fc5e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 10 Nov 2026 10:24:15 -0800 Subject: [PATCH 3/3] fix(moq-mux): a shed keeps a finished tail, rereads its sources, and drops queued leading frames Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/ts/export.rs | 21 +++-- rs/moq-mux/src/container/ts/schedule.rs | 5 ++ rs/moq-mux/src/jitter.rs | 112 +++++++++++++++++++----- 3 files changed, 109 insertions(+), 29 deletions(-) diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index fe1e9d35aac..571306b70f8 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -1810,8 +1810,12 @@ where // 4. Mux each frame the jitter buffer lets go (the first carries the buffered // PAT/PMT), then lay out every grid slot whose time has come ([`Self::lay_due`]). - if self.program.queue.is_empty() { - self.shed(); + // A shed makes room in each source's jitter window, so read them again. + if self.program.queue.is_empty() + && self.shed() + && let Err(err) = self.fill(waiter) + { + self.leave(err)?; } loop { if let Some(out) = self.pop() { @@ -3911,22 +3915,23 @@ where /// caught up: a frame held back while the output keeps up (for the program's tables, or a /// generation still going out) is no lag of the sink's, and a source that fell silent leaves /// nothing overdue. - fn shed(&mut self) { + /// Returns whether it shed. + fn shed(&mut self) -> bool { // Once every source has ended, the tail goes out late if it must. - let live = self.program.tracks.values().any(|track| track.read == Read::Live); - if self.program.mux_rate.is_none() || self.held.is_some() || !live { - return; + if self.program.mux_rate.is_none() || self.held.is_some() || self.program.schedule.ended() { + return false; } let (Some(due), Some(caught_up)) = (self.jitter.front_deadline(), self.caught_up) else { - return; + return false; }; let behind = kio::clock::now().saturating_duration_since(due.max(caught_up)); if behind <= self.budget() { - return; + return false; } tracing::warn!(?behind, "MPEG-TS output fell behind its multiplex rate, shedding"); self.jitter.shed(kio::clock::now()); self.rewind(); + true } /// Lay out every grid slot whose time has come, and wake when the next one is due. diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index 6d28bf9086b..19fa3b21fbd 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -689,6 +689,11 @@ impl Schedule { self.ended = ended; } + /// Whether every source has ended ([`Self::set_ended`]). + pub fn ended(&self) -> bool { + self.ended + } + /// Pad to `rate` bits per second from the next slot on, or stop padding. pub fn set_rate(&mut self, rate: Option) { if self.rate != rate { diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index feb374e8baa..a432e4da317 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -253,7 +253,7 @@ impl Buffer { ready.deadline, ready.item, ready.bytes, - ready.sync, + ready.place, )); track.bytes += ready.bytes; } @@ -432,7 +432,7 @@ pub(crate) struct Ready { /// Whether it is the first frame of its track to go out. pub first: bool, bytes: usize, - sync: bool, + place: Place, } /// What [`Buffer::push`] did with a frame. @@ -758,6 +758,13 @@ struct Floor { began: Instant, } +/// Where a queued frame sits in its group, so a shed can resume the track as [`Admit`] would. +#[derive(Clone, Copy)] +struct Place { + sync: bool, + leading: bool, +} + /// Which frames a track admits to its queue. #[derive(Clone, Copy, PartialEq, Eq, Debug)] enum Admit { @@ -772,8 +779,8 @@ enum Admit { struct Track { /// Frames in arrival order, each with its generation, decode time, deadline, bytes, and - /// whether it is a sync frame. - queue: VecDeque<(u64, Decode, Instant, T, usize, bool)>, + /// place in its group. + queue: VecDeque<(u64, Decode, Instant, T, usize, Place)>, /// Wake the reader when its future tail enters the admission window. read_timer: Option>>, /// What its queued frames hold against the byte budget. @@ -1090,9 +1097,17 @@ impl Engine { } let cost = arrival.cost(); track.bytes += cost; - track - .queue - .push_back((generation, arrival.decode, deadline, arrival.item, cost, arrival.sync)); + track.queue.push_back(( + generation, + arrival.decode, + deadline, + arrival.item, + cost, + Place { + sync: arrival.sync, + leading: arrival.leading, + }, + )); self.horizon = self.horizon.max(Some(deadline)); Push::Queued } @@ -1397,7 +1412,7 @@ impl Engine { && (self.delay.is_zero() || now >= *deadline) { let track = self.tracks.get_mut(key).expect("the front's track"); - let (generation, decode, deadline, item, bytes, sync) = + let (generation, decode, deadline, item, bytes, place) = track.queue.pop_front().expect("the front frame"); track.bytes -= bytes; let first = !std::mem::replace(&mut track.started, true); @@ -1414,7 +1429,7 @@ impl Engine { deadline, first, bytes, - sync, + place, }); } let Some(wake) = front.map(|(deadline, _)| deadline).into_iter().chain(acquired).min() else { @@ -1460,18 +1475,37 @@ impl Engine { /// Drop every frame due before `before`, which an output that fell that far behind can no /// longer send in time, and any read later that is. A track that loses a frame resumes at its - /// next sync frame. + /// next sync frame, queued or still to come, as if the frames had dropped as late. pub fn shed(&mut self, before: Instant) { self.shed = self.shed.max(Some(before)); for track in self.tracks.values_mut() { let mut lost = 0; - while let Some((_, _, deadline, _, bytes, sync)) = track.queue.front() - && (*deadline < before || (lost > 0 && !sync)) - { - track.bytes -= bytes; - track.queue.pop_front(); - lost += 1; - } + let mut admit = Admit::Flowing; + let bytes = &mut track.bytes; + track.queue.retain(|(_, _, deadline, _, cost, place)| { + let keep = match admit { + _ if *deadline < before => { + admit = Admit::Waiting; + false + } + Admit::Waiting if !place.sync => false, + Admit::Resumed if place.leading => false, + _ => { + if place.sync { + admit = match admit { + Admit::Waiting => Admit::Resumed, + Admit::Flowing | Admit::Resumed => Admit::Flowing, + }; + } + true + } + }; + if !keep { + lost += 1; + *bytes -= *cost; + } + keep + }); if lost == 0 { continue; } @@ -1479,10 +1513,7 @@ impl Engine { true => self.dropped += lost, false => self.trimmed += lost, } - track.admit = match track.queue.is_empty() { - true => Admit::Waiting, - false => Admit::Resumed, - }; + track.admit = admit; } } @@ -2267,6 +2298,45 @@ mod tests { assert_eq!(due(&mut buffer), ["v200", "v280", "v320", "v360"]); } + /// A shed drops what is due before the cut and resumes the track at its next sync frame + /// already queued, losing that frame's leading frames too, as a late frame would. + #[tokio::test(start_paused = true)] + async fn a_shed_resumes_at_the_next_queued_sync_frame() { + let start = Instant::now(); + let mut buffer = Buffer::new(DELAY).replay(); + let video = |decode, sync, leading, item| Arrival { + sync, + leading, + ..arrival(start, decode, item) + }; + buffer.push(1, video(0, true, false, "v0")).unwrap(); + tokio::time::advance(DELAY).await; + assert_eq!(due(&mut buffer), ["v0"]); + for (decode, sync, leading, item) in [ + (40, false, false, "v40"), + (80, false, false, "v80"), + (120, true, false, "v120"), + (160, false, true, "v160"), + (200, false, false, "v200"), + (240, true, false, "v240"), + (280, false, true, "v280"), + ] { + assert_eq!( + buffer.push(1, video(decode, sync, leading, item)).unwrap(), + Push::Queued + ); + } + + buffer.shed(start.saturating_add(DELAY + Duration::from_millis(100))); + assert_eq!( + buffer.dropped(), + 3, + "the stale frames and the resumed sync frame's leading frame" + ); + tokio::time::advance(Duration::from_secs(1)).await; + assert_eq!(due(&mut buffer), ["v120", "v200", "v240", "v280"]); + } + /// A forward restart before anything went out keeps the clock, so the tracks stay on one. #[tokio::test(start_paused = true)] async fn a_skip_before_the_first_release_keeps_one_clock() {