diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 00aefd0d5f8..387e1679dae 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -465,6 +465,9 @@ FLV, MKV and RTMP refuse the released `--max-age` spelling with a migration message. The output follows the source's clock within what ISO/IEC 13818-1 allows a TS PCR. A source further off fails the export, naming its rate, once it has used half the delay; a brief network stall only drops what it made late. +A timeline that steps further than the delay absorbs with no restart marker, as +a changeover between two encoders whose clocks disagree, re-acquires the clock +as at startup rather than read the step as drift. `export ts --passthrough` reads the track an `import ts --passthrough` published and writes its packets back out unchanged, from its first group on, each run diff --git a/quest/m1/ts-reliability/README.md b/quest/m1/ts-reliability/README.md index 2263286d940..ecd9a4cbbe5 100644 --- a/quest/m1/ts-reliability/README.md +++ b/quest/m1/ts-reliability/README.md @@ -100,7 +100,6 @@ catalog linger (#5305). ## Required - [A timebase step restarts the TS import](/quest/m1/ts-reliability/step-restart.md) - an unflagged step or a rewind republishes under a new epoch instead of ending the import and every subscriber's export -- [Export re-acquires a step](/quest/m1/ts-reliability/export-reacquire.md) - `moq export ts` re-acquires an unmarked timeline step instead of exiting on fake drift - [A restart keeps its catalog](/quest/m1/ts-reliability/restart-catalog.md) - a restarted import publishes its catalog with its first media - [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) - a running export follows a moved PMT PID - [A demoted subscriber priority reaches the publisher](/quest/m1/ts-reliability/stitch-catch-up.md) - a mid-stream priority demotion reaches the publisher on lite-07 and moq-transport, including an old instance after an epoch switch diff --git a/quest/m1/ts-reliability/export-reacquire.md b/quest/m1/ts-reliability/export-reacquire.md deleted file mode 100644 index 256991d7bd9..00000000000 --- a/quest/m1/ts-reliability/export-reacquire.md +++ /dev/null @@ -1,37 +0,0 @@ -# [S] The TS export re-acquires an unmarked timeline step - -## Goal - -`moq export ts` keeps running when its source's timestamps step with no -restart or skip marker. It re-acquires the clock as at startup, instead of -exiting with a drift error ("the source's clock runs +172.5 ppm off ours"). -This protects subscribers of TS publishers other than `moq import`. - -## Plan - -Traced 2026-10-28 from [#7102](https://github.com/moq-dev/moq/issues/7102): the -+172.5 ppm is noise, not a measurement of the source. - -- `jumps()` (`rs/moq-mux/src/jitter.rs`) opens a generation only on a marker. - An unmarked step goes to `steer()` on the old clock. -- The 600 s rate window drops every pre-step floor, but `startup` survives, - since only `acquired()` resets it. So `lost` reads as the whole step. -- The bail then fires on the first two-floor rate over 30 ppm (~345 µs of - arrival jitter across one 2 s step). - -Decided 2026-10-28 (maintainer): treat a step outside any marker as a new -acquisition, as `acquired()` does. Keep [#5564](https://github.com/moq-dev/moq/issues/5564)'s -decisions: no minimum span or sample count, and never widen the 30 ppm or -`lost` thresholds. A step is told apart from drift by size: a phase change no -clock rate within the bound could produce in the elapsed time. Check that the -straddling step's slew bound (`MAX_SLEW · elapsed`) cannot swing the drift -across the bound in one measurement. - -Tests: a forward and a backward unmarked step each re-acquire and keep -emitting. The #5564 startup cases still pass unchanged. - -Public API: none. Wire: none. - -## Related - -- [A timebase step restarts the TS import](/quest/m1/ts-reliability/step-restart.md) - the importer-side fix, under which `moq import` never forwards such a step diff --git a/quest/m1/ts-reliability/step-restart.md b/quest/m1/ts-reliability/step-restart.md index 57128f7f007..21dc595ca0f 100644 --- a/quest/m1/ts-reliability/step-restart.md +++ b/quest/m1/ts-reliability/step-restart.md @@ -57,6 +57,5 @@ Public API: the importer's error gains a restart cause. Wire: none. ## Related -- [Export re-acquires a step](/quest/m1/ts-reliability/export-reacquire.md) - the export-side defense for non-moq TS sources - [A restart keeps its catalog](/quest/m1/ts-reliability/restart-catalog.md) - removes the catalog hold a restart pays - [A stitched switch returns to --delay](/quest/m1/ts-reliability/stitch-catch-up.md) - drops the old instance's tail a restart leaves at each export diff --git a/rs/moq-mux/src/container/flv/export_test.rs b/rs/moq-mux/src/container/flv/export_test.rs index 4a6e495c0d3..a3ca9671555 100644 --- a/rs/moq-mux/src/container/flv/export_test.rs +++ b/rs/moq-mux/src/container/flv/export_test.rs @@ -1753,13 +1753,14 @@ async fn delay_drops_late_frames_and_drains_the_final_frame() { tokio::time::advance(std::time::Duration::from_millis(200)).await; assert!(export.poll_next(&kio::Waiter::noop()).is_pending()); live.track.write(raw_frame(20_000, b"expired", true)).unwrap(); - live.track.write(raw_frame(400_000, b"final", true)).unwrap(); + // Sent a delay ahead: any further would be a step in its timeline, which re-acquires. + live.track.write(raw_frame(300_000, b"final", true)).unwrap(); live.track.finish().unwrap(); live.catalog.finish().unwrap(); let final_frame = export.next().await.unwrap().unwrap(); assert!(final_frame.windows(5).any(|part| part == b"final")); assert!(!final_frame.windows(7).any(|part| part == b"expired")); - assert!(start.elapsed() >= std::time::Duration::from_millis(500)); + assert!(start.elapsed() >= std::time::Duration::from_millis(400)); assert!(export.next().await.unwrap().is_none()); } diff --git a/rs/moq-mux/src/container/mkv/export_test.rs b/rs/moq-mux/src/container/mkv/export_test.rs index 7c41893d1f2..e39f3e5b040 100644 --- a/rs/moq-mux/src/container/mkv/export_test.rs +++ b/rs/moq-mux/src/container/mkv/export_test.rs @@ -1157,13 +1157,14 @@ async fn delay_drops_late_frames_and_drains_the_final_frame() { tokio::time::advance(std::time::Duration::from_millis(200)).await; assert!(export.poll_next(&kio::Waiter::noop()).is_pending()); live.track.write(raw_frame(20_000, b"expired", true)).unwrap(); - live.track.write(raw_frame(400_000, b"final", true)).unwrap(); + // Sent a delay ahead: any further would be a step in its timeline, which re-acquires. + live.track.write(raw_frame(300_000, b"final", true)).unwrap(); live.track.finish().unwrap(); live.catalog.finish().unwrap(); let final_frame = export.next().await.unwrap().unwrap(); assert!(final_frame.windows(5).any(|part| part == b"final")); assert!(!final_frame.windows(7).any(|part| part == b"expired")); - assert!(start.elapsed() >= std::time::Duration::from_millis(500)); + assert!(start.elapsed() >= std::time::Duration::from_millis(400)); assert!(export.next().await.unwrap().is_none()); } diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index 0fe2c2de406..d351eeff091 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -86,6 +86,8 @@ struct Live { reorder: bool, /// Payload bytes in each non-key video frame. delta_size: usize, + /// Added to every timestamp written, in µs: changing it steps the timeline with no marker. + offset: i64, start: Instant, tick: u64, audio_index: u64, @@ -150,6 +152,7 @@ impl Live { audio_lag, reorder: false, delta_size: 400, + offset: 0, start: moq_clock::now(), tick: 0, audio_index: 0, @@ -268,7 +271,7 @@ impl Live { .write( group, &[Frame { - timestamp: Timestamp::from_micros(presented).unwrap(), + timestamp: Timestamp::from_micros(presented.strict_add_signed(self.offset)).unwrap(), duration: None, payload: length_prefixed(&[&slice]), keyframe, @@ -289,7 +292,7 @@ impl Live { let index = self.audio_index; self.audio .write(Frame { - timestamp: Timestamp::from_micros(index * AUDIO_US).unwrap(), + timestamp: Timestamp::from_micros((index * AUDIO_US).strict_add_signed(self.offset)).unwrap(), duration: None, payload: Bytes::from_iter((0..180u16).map(|i| (i ^ index as u16) as u8)), keyframe: index.is_multiple_of(AUDIO_GROUP), @@ -627,6 +630,33 @@ async fn a_track_sent_later_than_the_delay_loses_nothing() { ); } +/// A changeover between two encoders whose clocks started hours apart steps the whole timeline +/// forward with no marker, mid-group, as in the field. The export re-acquires its clock as at +/// startup, rather than read the step as drift and end, and renders on without a drop. (A step +/// back further than the group before its own never reaches it: the reader refuses it.) +#[tokio::test(start_paused = true)] +async fn an_unmarked_step_keeps_the_export_rendering() { + for step in [16_844_500_000, 2_000_000] { + for lag in [0, LAGS[0], LAGS[1]] { + let mut live = Live::new(1.0, lag).await; + let mut leg = live.join(Duration::ZERO).await; + live.run(8 * GOP + GOP / 2, &mut [&mut leg], |_| false).await; + live.offset += step; + let stepped = moq_clock::now(); + live.run(live.tick + 8 * GOP, &mut [&mut leg], |_| false).await; + + let case = format!("{} s step, lag {lag}", step as f64 / 1e6); + assert_eq!(leg.export.dropped(), 0, "{case}"); + let video = leg + .units() + .into_iter() + .filter(|unit| unit.pid == VIDEO_PID && unit.sent >= stepped) + .count() as u64; + assert!(video >= 6 * GOP, "{case}: only {video} video units after it"); + } + } +} + // A publisher that dies leaves the last video frames it sent held for their DTS, and nothing // settles them until frames come again. Where the receiver is not told the source changed (a // replacement on a protocol without an epoch, or the other half of a 1+1 pair), those come on the diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 5f368a81882..d985f446d3e 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -443,12 +443,16 @@ pub(crate) enum Push { /// whose clock is acquired the same way, though never ahead of a deadline already given /// out. So does a skip after which a frame would be held more than a delay past everything /// queued, beyond how far ahead of the others its track is sent: its timeline jumped ahead, -/// as when the skipped groups held a marker the publisher shed. How far ahead of its decode -/// time the source sends each track belongs to its mux and survives a restart, so the new -/// clock does not wait to hear from a track the previous one measured: it is anchored late -/// enough that the track sent latest keeps the delay when it gets there. A track that has -/// not reached the restart stays on the previous generation while its frames land there and -/// go out before the new generation's first frame; one that would follow it drops as late. +/// as when the skipped groups held a marker the publisher shed. So does a step with no marker +/// at all, as a changeover between two encoders whose clocks disagree, once the delay cannot +/// absorb it: read as drift it would fail [`Buffer::push`], and steering on it would let the +/// step's span of decode time slew the drift in one go. A smaller one is a move in latency. +/// How far ahead of its decode time the source sends each track belongs to its mux and +/// survives a restart, so the new clock does not wait to hear from a track the previous one +/// measured: it is anchored late enough that the track sent latest keeps the delay when it +/// gets there. A track that has not reached the restart stays on the previous generation +/// while its frames land there and go out before the new generation's first frame; one that +/// would follow it drops as late. /// One crossing the same restart joins the new generation when the frame lands on its /// clock, neither late nor held more than a delay past everything queued, and opens another /// otherwise. Before anything has gone out there is nothing to keep in step with, so a @@ -694,9 +698,12 @@ struct Floor { struct Track { /// Frames in arrival order, each with its generation, decode time and deadline. queue: VecDeque<(u64, Decode, Instant, T)>, - /// The source's restart and skip counters at the last frame. + /// The source's restart and skip counters at the last frame, when it decodes, and when it + /// arrived. restart: u64, skip: u64, + decode: Decode, + arrived: Instant, /// The generation of the last frame. generation: u64, /// Frames are dropped until the next sync frame. @@ -822,9 +829,20 @@ impl Engine { /// Whether a frame moves its track off the clock: a restart, unless the track is crossing /// one another track already opened a generation for and the frame lands on its clock, or - /// a frame held past [`Self::bound`] after a skip, or while a generation is acquired. The - /// latter is a track reaching the new timeline without a marker of its own, which belongs - /// to that generation, not far ahead on the old clock. + /// a frame held past [`Self::bound`] after a skip, or while a generation is acquired, or a + /// step in its timeline with no marker that the delay cannot absorb. The second is a track + /// reaching the new timeline without a marker of its own, which belongs to that generation, + /// not far ahead on the old clock. + /// + /// An unmarked step is told apart from drift and queueing by size. Forward, a frame is held + /// past [`Self::bound`] and reached us more than a delay earlier against its decode time + /// than its track's last: no clock within [`MAX_DRIFT`] moves it that far in the time + /// between them, and frames reach us in the order they are sent, so neither a backlog + /// draining nor a track whose frames are sparse does. Back, a frame decodes before its + /// track's last and is late: a stall only delays what comes next. It is a step on the + /// clock's own generation, since a track still on an earlier one is on that timeline. + /// Held for the next generation, the frame never steers the old clock. Without a delay + /// nothing is held or late, so a step costs nothing there. fn jumps(&self, key: &K, arrival: &Arrival) -> bool { let (Some(clock), Some(track)) = (self.clock, self.tracks.get(key)) else { return false; @@ -832,9 +850,15 @@ impl Engine { let deadline = self.deadline(&clock, arrival.decode); let restarted = track.restart != arrival.restart && (track.generation == clock.generation || !self.lands(key, deadline, arrival.arrived)); - let leapt = (track.skip != arrival.skip || self.acquire.is_some()) - && deadline.is_some_and(|deadline| deadline > self.bound(key, arrival.arrived)); - restarted || leapt + let held = deadline.is_some_and(|deadline| deadline > self.bound(key, arrival.arrived)); + let leapt = (track.skip != arrival.skip || self.acquire.is_some()) && held; + let unmarked = !self.delay.is_zero() && track.generation == clock.generation; + let elapsed = arrival.decode.as_nanos() - track.decode.as_nanos(); + let ahead = elapsed - since(track.arrived, arrival.arrived); + let drift = (elapsed.abs() as f64 * MAX_DRIFT) as i128; + let forward = held && ahead > self.delay.as_nanos() as i128 + drift; + let back = elapsed < 0 && deadline.is_none_or(|deadline| deadline < arrival.arrived); + restarted || leapt || (unmarked && (forward || back)) } /// Hold a frame for the generation being acquired, starting one if none is, and anchor it @@ -928,6 +952,8 @@ impl Engine { queue: VecDeque::new(), restart: arrival.restart, skip: arrival.skip, + decode: arrival.decode, + arrived: arrival.arrived, generation, waiting: false, resumed: false, @@ -935,6 +961,8 @@ impl Engine { }); track.restart = arrival.restart; track.skip = arrival.skip; + track.decode = arrival.decode; + track.arrived = arrival.arrived; track.generation = generation; let push = match deadline { _ if stale => { @@ -2682,6 +2710,103 @@ mod tests { } } + /// What [`unmarked`] saw. + struct Stepped { + err: Option, + dropped: u64, + /// Each frame's generation, in the order they went out. + out: Vec, + /// The clock's drift just before the step, and at the end. + before: f64, + after: f64, + } + + /// Feed `secs` of 10 fps frames of `shape`'s tracks through its buffer from a source on our + /// clock, each sent as it decodes with a few ms of jitter that changes every steering step, + /// whose timeline steps by `step` seconds at `at` s with no restart or skip marker. + async fn unmarked(shape: &Shape, at: f64, step: f64, secs: f64) -> Stepped { + // Where the timeline starts, so a step back stays on it. + const BASE: f64 = 20_000.0; + let start = moq_clock::now(); + let clock = moq_clock::Deadlines::new(start); + let _entered = clock.enter(); + let mut buffer = Buffer::new(shape.delay); + buffer.expect(shape.tracks.iter().copied()); + let mut stepped = Stepped { + err: None, + dropped: 0, + out: Vec::new(), + before: 0.0, + after: 0.0, + }; + for k in 0..(secs * 10.0) as u64 { + let sent = k as f64 / 10.0; + let jitter = ((k / 20) * 3) % 5; + let arrived = start + Duration::from_secs_f64(sent + jitter as f64 * 0.002); + clock.advance(arrived.max(clock.now())); + if sent < at { + stepped.before = buffer.clock.map_or(0.0, |clock| clock.drift); + } + let decode = BASE + sent + if sent >= at { step } else { 0.0 }; + for &track in shape.tracks { + let frame = Arrival { + arrived, + read: arrived, + decode: Timestamp::from_micros((decode * 1e6).round() as u64).unwrap().into(), + restart: 0, + skip: 0, + sync: true, + leading: false, + item: "v", + }; + if let Err(err) = buffer.push(track, frame) { + stepped.err = Some(err); + return stepped; + } + } + stepped + .out + .extend(released(&mut buffer).into_iter().map(|(_, generation)| generation)); + } + stepped.dropped = buffer.dropped(); + stepped.after = buffer.clock.map_or(0.0, |clock| clock.drift); + stepped + } + + /// A timeline that steps with no marker, as a changeover between two encoders whose clocks + /// started hours apart, by more than the delay absorbs, either way, before the rate window + /// fills or once it has, re-acquires the clock instead of reading the step as drift: one + /// new generation, nothing dropped, every frame out, and the step never swings the drift + /// further than the slew limit allows over the time that passed. + #[tokio::test(start_paused = true)] + async fn an_unmarked_step_reacquires_the_clock() { + for at in [30.0, 1_200.0] { + for step in [16_844.5, -16_844.5, 2.0, -2.0] { + let secs = at + 60.0; + let run = unmarked(&PAIR, at, step, secs).await; + let case = format!("{step:+} s step at {at} s"); + assert!(run.err.is_none(), "{case}: {:?}", run.err); + assert_eq!(run.dropped, 0, "{case}"); + assert!(run.out.is_sorted(), "{case}: a generation went out after the next"); + assert_eq!(run.out.last(), Some(&1), "{case}: no single new generation"); + // What is still held at the end goes out within a delay. + let held = (PAIR.delay.as_secs_f64() * 10.0).ceil() as usize + 1; + let pushed = (secs * 10.0) as usize * PAIR.tracks.len(); + assert!( + run.out.len() + held * PAIR.tracks.len() >= pushed, + "{case}: {} of {pushed} went out", + run.out.len() + ); + let swing = (run.after - run.before).abs(); + assert!( + swing <= MAX_SLEW * (secs - at) * (1.0 + 1e-9), + "{case}: the drift swung {} ppm", + swing * 1e6 + ); + } + } + } + /// A source within what the clock may follow never fails, though its first frames reached /// us late, so the clock was anchored that much late, and each frame jitters a few ms, so /// the first floors, a few ms apart over two seconds, measure a rate far past the limit.