diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index c30fabb339e..1bb43b84d2f 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -104,7 +104,6 @@ remain with their current quests. - [Access units](/quest/m1/ts-reliability/pes-access-units.md) - a PES carrying several access units publishes each at its own time - [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) - a revised PAT never leaves a broadcast meaning another program - [Stream routes](/quest/m1/ts-reliability/stream-routes.md) - every PID gets a declared route, and what is not carried is counted -- [Release clock false alarms](/quest/m1/ts-reliability/release-clock-false-alarms.md) - no false jitter leap at a 500 ms delay and no false drift alarm - [Program cap](/quest/m1/ts-reliability/program-cap.md) - a PAT listing every program number cannot create a broadcast per program ## Related diff --git a/quest/m1/ts-reliability/release-clock-false-alarms.md b/quest/m1/ts-reliability/release-clock-false-alarms.md deleted file mode 100644 index c2599c4044d..00000000000 --- a/quest/m1/ts-reliability/release-clock-false-alarms.md +++ /dev/null @@ -1,26 +0,0 @@ -# [S] The TS export release clock raises no false alarms - -## Goal - -The TS export's jitter buffer reports a jitter leap or an `out_of_tolerance` -drift only when the source clock actually misbehaves. The recovery arms of -[moq-dev/moq#6139](https://github.com/moq-dev/moq/pull/6139) run clean at a -500 ms export delay and across the long arm, and assert it. - -## Plan - -Decided 2026-10-21 with the maintainer: m1, under TS reliability. Both seen -in #6139, whose arms stayed at 1 s (`export ts --delay`'s default) to land: - -- **Jitter leap at 500 ms.** A declared PCR discontinuity rewinding 475 ms - after a skipped group reads as a jitter leap. A declared discontinuity is - a new timeline, not jitter. -- **Drift estimate.** In the long arm the rate estimate read 5,551 ppm once, - at about 95 s, counting one `out_of_tolerance` against an exact source - clock. The final estimate was 0.00015 ppm and output was unaffected, but - anyone monitoring the counter sees a false alarm. Find which sample - produced it before changing the estimator. - -Reproduce each first, then make the arms assert both counters stay zero. - -Public API: none. Wire: none. diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 06163cb48a9..bb20a3944f0 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -469,8 +469,20 @@ impl Track { } self.queue(name, pending, input.jitter)?; } - // Under sync the frames read settle each DTS, not when this leg found the source - // empty, and [`Self::queue`] drops what a cut group left held. + // A cut takes a start group it left short now, not with the next group, which may + // arrive after that start was due, or with the stream's end. Under sync the frames + // read settle each DTS, not when this leg found the source empty, so what the cut + // group left held goes, as on a leg that never read it. + Poll::Ready(Some(Event::Cut)) if !input.waiting_for_header => { + match self.sync { + None => { + self.settle = false; + self.release(name, input.jitter, true)?; + } + Some(_) => self.clock.discard(), + } + input.jitter.cut(&self.pid, self.base.2 + self.source.cuts()); + } Poll::Ready(Some(Event::GroupEnd | Event::Cut)) => self.settle = self.sync.is_none(), Poll::Ready(Some(Event::FrameEnd { .. })) => {} Poll::Ready(None) => { diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index a5e8a929d76..1a81554710e 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -861,9 +861,11 @@ async fn a_source_past_the_limit_is_counted() { /// A source 22% slow, as an importer starved of CPU, fails the export within seconds. Past the /// limit, a source fails once it has used half the delay however far off it runs; the jitter -/// buffer's own tests time that at 40 and 400 ppm. +/// buffer's own tests time that at 40 and 400 ppm. On the few floors there are by then, the +/// video's wandering lead fits its envelope as well as the audio does, and the one measuring +/// less drift names the rate, so only its order is checked. #[tokio::test(start_paused = true)] -#[should_panic(expected = "the source's clock runs -220")] +#[should_panic(expected = "the source's clock runs -2")] async fn a_source_far_off_real_time_fails_within_seconds() { let mut live = Live::new(1.0 / 1.22, 0).await; let mut leg = live.join(Duration::ZERO).await; diff --git a/rs/moq-mux/src/container/ts/recovery_test.rs b/rs/moq-mux/src/container/ts/recovery_test.rs index fd3daeed63c..063f1903fe7 100644 --- a/rs/moq-mux/src/container/ts/recovery_test.rs +++ b/rs/moq-mux/src/container/ts/recovery_test.rs @@ -42,8 +42,9 @@ const FRAME: Duration = Duration::from_millis(40); const AUDIO_TICKS: u64 = 1920; const WRAP: u64 = 1 << 33; -/// How long the export holds each frame past its decode time: `export ts --delay`'s default. -const DELAY: Duration = Duration::from_secs(1); +/// How long the export holds each frame past its decode time: half `export ts --delay`'s +/// default, so a fault costs more of it. +const DELAY: Duration = Duration::from_millis(500); /// How long after its nominal time a frame arrives on a clean path, about a frame's transit. const LAG: Duration = Duration::from_millis(40); /// How far a frame's lateness may sit from a bound before the ledger calls it either way: the @@ -1292,6 +1293,11 @@ async fn recover(arm: Arm) -> Run { let run = run(&recording, &arm).await; let _evidence = Evidence::save(&run); grade(&run.ledgers, &run.stats).unwrap(); + assert_eq!( + run.stats.release.out_of_tolerance, 0, + "exact source clock: {:?}", + run.stats.release + ); assert!(assert_on_time(&run.frames) > 200, "too little output to judge"); assert_continuous(&run.ts()); assert_eq!( @@ -1379,8 +1385,9 @@ async fn a_cut_group_keeps_what_it_delivered() { } /// The video stalls for 2.5 s while the audio runs on, then delivers what it held at once: the -/// source skips the groups it held past the age budget as one gap, nothing late goes out, and the -/// video resumes on the first keyframe delivered in time, on the same clock. +/// source skips the groups it held past the age budget as one gap, the newest whole group it held, +/// within that budget but past its deadline, is dropped, nothing late goes out, and the video +/// resumes on the first keyframe delivered in time, on the same clock. #[tokio::test(start_paused = true)] async fn a_stall_drops_what_it_made_late() { let run = recover(arm(Profile::Closed, |slot| hold(slot.video, slot))).await; @@ -1389,7 +1396,7 @@ async fn a_stall_drops_what_it_made_late() { video.units.iter().any(|unit| unit.on_time == Some(false)), "the schedule made pictures late" ); - assert_eq!((run.stats.release.dropped, run.stats.release.gaps), (0, 1)); + assert_eq!((run.stats.release.dropped, run.stats.release.gaps), (12, 1)); let first = audio.units.iter().position(|unit| unit.output).unwrap(); assert!( audio.units[first..].iter().all(|unit| unit.output), diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index b3d6bbc27a3..7495047a5ef 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -89,11 +89,11 @@ pub(crate) enum Push { /// marker) has moved it, so once frames have gone out the restart opens a new generation, /// whose clock is acquired the same way, though never ahead of a deadline already given /// out. So does a skip after which a frame would be held more than a delay past everything -/// queued: its timeline jumped ahead, as when the skipped groups held a marker the -/// publisher shed. How far ahead of its decode time the source sends each track belongs to -/// its mux and survives a restart, so the new clock does not wait to hear from a track the -/// previous one measured: it is anchored late enough that the track sent latest keeps the -/// delay when it gets there. A track that has +/// queued, beyond how far ahead of the others its track is sent: its timeline jumped ahead, +/// as when the skipped groups held a marker the publisher shed. How far ahead of its decode +/// time the source sends each track belongs to its mux and survives a restart, so the new +/// clock does not wait to hear from a track the previous one measured: it is anchored late +/// enough that the track sent latest keeps the delay when it gets there. A track that has /// not reached the restart stays on the previous generation while its frames land there and /// go out before the new generation's first frame; one that would follow it drops as late. /// One crossing the same restart joins the new generation when the frame lands on its @@ -189,6 +189,9 @@ const RESPONSE: f64 = 600.0; const MAX_DRIFT: f64 = 810.0 / 27e6; /// How fast that may change, per second: 0.075 Hz/s of 27 MHz (ISO/IEC 13818-1 2.4.2.1). const MAX_SLEW: f64 = 0.075 / 27e6; +/// How close two tracks' depths under their envelopes are to fit them equally, in seconds: a +/// nanosecond, the resolution of the times they are measured from. +const FIT: f64 = 1e-9; /// The decode timeline's mapping onto ours: where a frame decoding at `base` is due, less the /// delay, and how much longer than the decode timeline ours runs. @@ -213,6 +216,8 @@ struct Acquire { tracks: BTreeMap, /// A track crossed a group boundary while these frames were being read. grouped: bool, + /// Each track's cut count reported by [`Buffer::cut`] while these frames were held. + cuts: BTreeMap, } impl Acquire { @@ -294,19 +299,28 @@ impl Steer { /// whichever track in `heard`, of those heard in at least half the floors, lies closest /// under its own at the median, so a spell of queueing on it shorter than half the window /// does not hand the rate to another; the phase is the floors' envelope at that slope. + /// + /// Of tracks that fit their envelopes equally, as any with only two floors does, the one + /// measuring the least drift is taken: a track whose send-ahead wanders fits as well as one + /// that holds steady, and only the source's clock moves every track. fn measure<'a>(&self, heard: impl Iterator, now: f64) -> Option<(f64, f64)> where K: 'a, { - let steadiest = heard + let fits: Vec<(f64, f64)> = heard .filter_map(|key| self.tracks.get(key)) .filter(|series| 2 * series.len() >= self.phase.len()) .filter_map(|series| { let (slope, at) = envelope(series, now)?; Some((depth(series, slope, at, now), slope)) }) - .min_by(|a, b| a.0.total_cmp(&b.0)) - .map(|(_, slope)| slope); + .collect(); + let best = fits.iter().map(|&(depth, _)| depth).fold(f64::INFINITY, f64::min); + let steadiest = fits + .iter() + .filter(|&&(depth, _)| depth - best <= FIT) + .map(|&(_, slope)| slope) + .min_by(|a, b| a.abs().total_cmp(&b.abs())); let rate = match steadiest { Some(rate) => rate, None => envelope(&self.phase, now)?.0, @@ -344,6 +358,19 @@ struct Track { started: bool, } +impl Track { + /// Take the source's cut count. The rest of a group cut since will never arrive, so until a + /// frame of the track has gone out, what it holds goes too. Returns how many frames that was. + fn cut(&mut self, cut: u64) -> u64 { + let lost = match cut != self.cut && !self.started { + true => std::mem::take(&mut self.queue).len() as u64, + false => 0, + }; + self.cut = cut; + lost + } +} + impl Buffer { pub fn new(delay: Duration) -> Self { Self { @@ -419,7 +446,7 @@ impl Buffer { .filter(|track| track.generation != clock.generation && track.restart == arrival.restart); if let Some(track) = behind && !self.delay.is_zero() - && !self.lands(deadline, arrival.arrived) + && !self.lands(&key, deadline, arrival.arrived) { let generation = track.generation; return Ok(self.queue(key, arrival, generation, None, false)); @@ -440,7 +467,7 @@ impl Buffer { let deadline = self.deadline(&previous, arrival.decode); let lags = track.generation == previous.generation && track.restart == arrival.restart - && self.lands(deadline, arrival.arrived) + && self.lands(key, deadline, arrival.arrived) && deadline.is_some_and(|deadline| self.opened().is_none_or(|first| deadline <= first)); lags.then_some(previous) } @@ -458,18 +485,18 @@ impl Buffer { /// Whether a frame moves its track off the clock: a restart, unless the track is crossing /// one another track already opened a generation for and the frame lands on its clock, or - /// a frame held more than a delay past everything queued after a skip, or while a - /// generation is acquired. The latter is a track reaching the new timeline without a - /// marker of its own, which belongs to that generation, not far ahead on the old clock. + /// a frame held past [`Self::bound`] after a skip, or while a generation is acquired. The + /// latter is a track reaching the new timeline without a marker of its own, which belongs + /// to that generation, not far ahead on the old clock. fn jumps(&self, key: &K, arrival: &Arrival) -> bool { let (Some(clock), Some(track)) = (self.clock, self.tracks.get(key)) else { return false; }; let deadline = self.deadline(&clock, arrival.decode); let restarted = track.restart != arrival.restart - && (track.generation == clock.generation || !self.lands(deadline, arrival.arrived)); + && (track.generation == clock.generation || !self.lands(key, deadline, arrival.arrived)); let leapt = (track.skip != arrival.skip || self.acquire.is_some()) - && deadline.is_some_and(|deadline| deadline > self.bound(arrival.arrived)); + && deadline.is_some_and(|deadline| deadline > self.bound(key, arrival.arrived)); restarted || leapt } @@ -484,6 +511,7 @@ impl Buffer { frames: Vec::new(), tracks: BTreeMap::new(), grouped: false, + cuts: BTreeMap::new(), }); let mid = acquire.tracks.entry(key.clone()).or_default(); acquire.grouped |= arrival.sync && *mid; @@ -540,6 +568,9 @@ impl Buffer { let (deadline, stale) = self.judge(&clock, &frame, acquiring); self.queue(key, frame, clock.generation, deadline, stale); } + for (key, cut) in acquire.cuts { + self.cut(&key, cut); + } } /// A held frame's deadline on `clock`, and whether it was due before it was read, when @@ -571,14 +602,9 @@ impl Buffer { resumed: false, started: false, }); - // The rest of the group the track holds will never arrive. - if arrival.cut != track.cut && !track.started { - self.trimmed += track.queue.len() as u64; - track.queue.clear(); - } + self.trimmed += track.cut(arrival.cut); track.restart = arrival.restart; track.skip = arrival.skip; - track.cut = arrival.cut; track.generation = generation; let push = match deadline { _ if stale => { @@ -627,16 +653,19 @@ impl Buffer { } } - /// Whether a frame arriving at `arrived` and due at `deadline` is on the clock: not late, - /// and held no more than a delay past both its arrival and everything already queued. - fn lands(&self, deadline: Option, arrived: Instant) -> bool { - deadline.is_some_and(|deadline| arrived <= deadline && deadline <= self.bound(arrived)) + /// Whether a frame of `key` arriving at `arrived` and due at `deadline` is on the clock: not + /// late, and held no more than [`Self::bound`] allows. + fn lands(&self, key: &K, deadline: Option, arrived: Instant) -> bool { + deadline.is_some_and(|deadline| arrived <= deadline && deadline <= self.bound(key, arrived)) } - /// The latest a frame arriving at `arrived` may be due and still be on the clock: a delay - /// past both its arrival and everything already queued. - fn bound(&self, arrived: Instant) -> Instant { - arrived.max(self.horizon.unwrap_or(arrived)) + self.delay + /// The latest a frame of `key` arriving at `arrived` may be due and still be on the clock: a + /// delay past both its arrival and everything already queued, and its track's lead further. + /// A track sent ahead of the others is held that much longer, so once a stall of its own + /// leaves only theirs queued, its next frame is not taken for a leap. + fn bound(&self, key: &K, arrived: Instant) -> Instant { + let lead = Duration::from_secs_f64(self.lead(key).unwrap_or(0.0).max(0.0)); + arrived.max(self.horizon.unwrap_or(arrived)) + self.delay + lead } /// Account for the slack a frame arrived with, and every [`STEER`] of our time step the @@ -913,6 +942,22 @@ impl Buffer { } } + /// The source lost the rest of the group `key` was in, its `cut`th such loss + /// ([`Arrival::cut`]), known before the track's next frame: what a track that has not + /// started holds goes with it now, rather than once that frame arrives, which may be after + /// the cut-off start was due. + pub fn cut(&mut self, key: &K, cut: u64) { + if let Some(acquire) = &mut self.acquire + && acquire.tracks.contains_key(key) + { + acquire.cuts.insert(key.clone(), cut); + return; + } + if let Some(track) = self.tracks.get_mut(key) { + self.trimmed += track.cut(cut); + } + } + /// Count a frame the caller dropped after [`Self::poll_next`] released it. The first of its /// track is trimmed and leaves the track not started; a later one is a loss. pub fn discard(&mut self, ready: &Ready) { @@ -1064,6 +1109,22 @@ mod tests { const DELAY: Duration = Duration::from_millis(100); + /// Two floors from the recovery arm's exact audio clock and wandering video lead. + /// Both envelopes interpolate their two samples, so rounding in their equal depths + /// must not make the changing send-ahead look like source clock drift. + #[test] + fn equal_depths_do_not_turn_a_track_lead_into_clock_drift() { + let audio = VecDeque::from([(97.0, 9.44999008545876e-10), (98.32, 4.143163449267329e-10)]); + let video = VecDeque::from([(98.726666, 0.6206660009629188), (99.004, 0.5390000015665989)]); + let steer = Steer { + phase: audio.clone(), + tracks: BTreeMap::from([(1, audio), (2, video)]), + ..Steer::default() + }; + let (rate, _) = steer.measure([1, 2].iter(), 100.284).unwrap(); + assert!(rate.abs() < MAX_DRIFT, "an exact clock measured {rate} off real time"); + } + fn ms(ms: u64) -> Timestamp { Timestamp::from_millis(ms).unwrap() } @@ -1554,6 +1615,73 @@ mod tests { assert_eq!(released(&mut buffer), [("a5000", 1)]); } + /// A track sent ahead of the others is held its lead past the delay. Once a skip of its own + /// leaves only theirs queued, its next frame is due that lead past them, and keeps the clock. + #[tokio::test(start_paused = true)] + async fn a_track_sent_ahead_resuming_after_a_skip_keeps_the_clock() { + const LEAD: u64 = 300; + let start = moq_clock::now(); + let mut buffer = Buffer::new(DELAY); + buffer.expect([1, 2]); + let mut out = Vec::new(); + for at in (0..12_000).step_by(40) { + tokio::time::advance(start + Duration::from_millis(at) - moq_clock::now()).await; + let now = moq_clock::now(); + assert_eq!(buffer.push(1, arrival(now, at, "v")).unwrap(), Push::Queued); + // Track 2 is sent LEAD ahead, and skips a second of its own. + let decode = at + LEAD; + if !(8_000..9_000).contains(&decode) { + let skip = u64::from(decode >= 9_000); + let frame = Arrival { + skip, + ..arrival(now, decode, "a") + }; + assert_eq!(buffer.push(2, frame).unwrap(), Push::Queued, "at {at} ms"); + } + out.extend(released(&mut buffer)); + } + assert!(out.len() > 500, "too little went out: {}", out.len()); + assert!( + out.iter().all(|&(_, generation)| generation == 0), + "a generation opened" + ); + assert_eq!(buffer.dropped(), 0); + } + + /// A cut the source reports before the track's next frame takes the start group it left + /// short then, whether the clock is still being acquired or the group is queued, though the + /// next group arrives only after that start was due. + #[tokio::test(start_paused = true)] + async fn a_cut_reported_before_the_next_group_drops_the_start_group() { + for replay in [false, true] { + let mut buffer = Buffer::new(DELAY); + if replay { + buffer = buffer.replay(); + } + let video = |decode, sync, cut, item| Arrival { + sync, + skip: cut, + cut, + ..arrival(moq_clock::now(), decode, item) + }; + buffer.push(1, video(0, true, 0, "v0")).unwrap(); + tokio::time::advance(Duration::from_millis(40)).await; + buffer.push(1, video(40, false, 0, "v40")).unwrap(); + tokio::time::advance(Duration::from_millis(20)).await; + buffer.cut(&1, 1); + tokio::time::advance(DELAY).await; + assert_eq!( + due(&mut buffer), + Vec::<&str>::new(), + "replay {replay}: the cut-off start went out" + ); + buffer.push(1, video(160, true, 1, "v160")).unwrap(); + tokio::time::advance(2 * DELAY).await; + assert_eq!(due(&mut buffer), ["v160"], "replay {replay}"); + assert_eq!((buffer.dropped(), buffer.trimmed()), (0, 2), "replay {replay}"); + } + } + /// Every frame of a generation goes out before the next generation's, even one the new /// clock puts earlier. #[tokio::test(start_paused = true)]