Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions doc/bin/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,9 @@ FLV, MKV and RTMP refuse the released `--max-age` spelling with a migration
message. The output follows the source's clock within what ISO/IEC 13818-1
allows a TS PCR. A source further off fails the export, naming its rate, once
it has used half the delay; a brief network stall only drops what it made late.
A timeline that steps further than the delay absorbs with no restart marker, as
a changeover between two encoders whose clocks disagree, re-acquires the clock
as at startup rather than read the step as drift.

`export ts --passthrough` reads the track an `import ts --passthrough` published
and writes its packets back out unchanged, from its first group on, each run
Expand Down
1 change: 0 additions & 1 deletion quest/m1/ts-reliability/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,6 @@ catalog linger (#5305).
## Required

- [A timebase step restarts the TS import](/quest/m1/ts-reliability/step-restart.md) - an unflagged step or a rewind republishes under a new epoch instead of ending the import and every subscriber's export
- [Export re-acquires a step](/quest/m1/ts-reliability/export-reacquire.md) - `moq export ts` re-acquires an unmarked timeline step instead of exiting on fake drift
- [A restart keeps its catalog](/quest/m1/ts-reliability/restart-catalog.md) - a restarted import publishes its catalog with its first media
- [PAT changes](/quest/m1/ts-reliability/pat-lifecycle.md) - a running export follows a moved PMT PID
- [A demoted subscriber priority reaches the publisher](/quest/m1/ts-reliability/stitch-catch-up.md) - a mid-stream priority demotion reaches the publisher on lite-07 and moq-transport, including an old instance after an epoch switch
Expand Down
37 changes: 0 additions & 37 deletions quest/m1/ts-reliability/export-reacquire.md

This file was deleted.

1 change: 0 additions & 1 deletion quest/m1/ts-reliability/step-restart.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,5 @@ Public API: the importer's error gains a restart cause. Wire: none.

## Related

- [Export re-acquires a step](/quest/m1/ts-reliability/export-reacquire.md) - the export-side defense for non-moq TS sources
- [A restart keeps its catalog](/quest/m1/ts-reliability/restart-catalog.md) - removes the catalog hold a restart pays
- [A stitched switch returns to --delay](/quest/m1/ts-reliability/stitch-catch-up.md) - drops the old instance's tail a restart leaves at each export
5 changes: 3 additions & 2 deletions rs/moq-mux/src/container/flv/export_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1753,13 +1753,14 @@ async fn delay_drops_late_frames_and_drains_the_final_frame() {
tokio::time::advance(std::time::Duration::from_millis(200)).await;
assert!(export.poll_next(&kio::Waiter::noop()).is_pending());
live.track.write(raw_frame(20_000, b"expired", true)).unwrap();
live.track.write(raw_frame(400_000, b"final", true)).unwrap();
// Sent a delay ahead: any further would be a step in its timeline, which re-acquires.
live.track.write(raw_frame(300_000, b"final", true)).unwrap();
live.track.finish().unwrap();
live.catalog.finish().unwrap();
let final_frame = export.next().await.unwrap().unwrap();
assert!(final_frame.windows(5).any(|part| part == b"final"));
assert!(!final_frame.windows(7).any(|part| part == b"expired"));
assert!(start.elapsed() >= std::time::Duration::from_millis(500));
assert!(start.elapsed() >= std::time::Duration::from_millis(400));
assert!(export.next().await.unwrap().is_none());
}

Expand Down
5 changes: 3 additions & 2 deletions rs/moq-mux/src/container/mkv/export_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1157,13 +1157,14 @@ async fn delay_drops_late_frames_and_drains_the_final_frame() {
tokio::time::advance(std::time::Duration::from_millis(200)).await;
assert!(export.poll_next(&kio::Waiter::noop()).is_pending());
live.track.write(raw_frame(20_000, b"expired", true)).unwrap();
live.track.write(raw_frame(400_000, b"final", true)).unwrap();
// Sent a delay ahead: any further would be a step in its timeline, which re-acquires.
live.track.write(raw_frame(300_000, b"final", true)).unwrap();
live.track.finish().unwrap();
live.catalog.finish().unwrap();
let final_frame = export.next().await.unwrap().unwrap();
assert!(final_frame.windows(5).any(|part| part == b"final"));
assert!(!final_frame.windows(7).any(|part| part == b"expired"));
assert!(start.elapsed() >= std::time::Duration::from_millis(500));
assert!(start.elapsed() >= std::time::Duration::from_millis(400));
assert!(export.next().await.unwrap().is_none());
}

Expand Down
34 changes: 32 additions & 2 deletions rs/moq-mux/src/container/ts/export_timing_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ struct Live {
reorder: bool,
/// Payload bytes in each non-key video frame.
delta_size: usize,
/// Added to every timestamp written, in µs: changing it steps the timeline with no marker.
offset: i64,
start: Instant,
tick: u64,
audio_index: u64,
Expand Down Expand Up @@ -150,6 +152,7 @@ impl Live {
audio_lag,
reorder: false,
delta_size: 400,
offset: 0,
start: moq_clock::now(),
tick: 0,
audio_index: 0,
Expand Down Expand Up @@ -268,7 +271,7 @@ impl Live {
.write(
group,
&[Frame {
timestamp: Timestamp::from_micros(presented).unwrap(),
timestamp: Timestamp::from_micros(presented.strict_add_signed(self.offset)).unwrap(),
duration: None,
payload: length_prefixed(&[&slice]),
keyframe,
Expand All @@ -289,7 +292,7 @@ impl Live {
let index = self.audio_index;
self.audio
.write(Frame {
timestamp: Timestamp::from_micros(index * AUDIO_US).unwrap(),
timestamp: Timestamp::from_micros((index * AUDIO_US).strict_add_signed(self.offset)).unwrap(),
duration: None,
payload: Bytes::from_iter((0..180u16).map(|i| (i ^ index as u16) as u8)),
keyframe: index.is_multiple_of(AUDIO_GROUP),
Expand Down Expand Up @@ -627,6 +630,33 @@ async fn a_track_sent_later_than_the_delay_loses_nothing() {
);
}

/// A changeover between two encoders whose clocks started hours apart steps the whole timeline
/// forward with no marker, mid-group, as in the field. The export re-acquires its clock as at
/// startup, rather than read the step as drift and end, and renders on without a drop. (A step
/// back further than the group before its own never reaches it: the reader refuses it.)
#[tokio::test(start_paused = true)]
async fn an_unmarked_step_keeps_the_export_rendering() {
for step in [16_844_500_000, 2_000_000] {
for lag in [0, LAGS[0], LAGS[1]] {
let mut live = Live::new(1.0, lag).await;
let mut leg = live.join(Duration::ZERO).await;
live.run(8 * GOP + GOP / 2, &mut [&mut leg], |_| false).await;
live.offset += step;
let stepped = moq_clock::now();
live.run(live.tick + 8 * GOP, &mut [&mut leg], |_| false).await;

let case = format!("{} s step, lag {lag}", step as f64 / 1e6);
assert_eq!(leg.export.dropped(), 0, "{case}");
let video = leg
.units()
.into_iter()
.filter(|unit| unit.pid == VIDEO_PID && unit.sent >= stepped)
.count() as u64;
assert!(video >= 6 * GOP, "{case}: only {video} video units after it");
}
}
}

// A publisher that dies leaves the last video frames it sent held for their DTS, and nothing
// settles them until frames come again. Where the receiver is not told the source changed (a
// replacement on a protocol without an epoch, or the other half of a 1+1 pair), those come on the
Expand Down
151 changes: 138 additions & 13 deletions rs/moq-mux/src/jitter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -443,12 +443,16 @@ pub(crate) enum Push {
/// whose clock is acquired the same way, though never ahead of a deadline already given
/// out. So does a skip after which a frame would be held more than a delay past everything
/// queued, beyond how far ahead of the others its track is sent: its timeline jumped ahead,
/// as when the skipped groups held a marker the publisher shed. How far ahead of its decode
/// time the source sends each track belongs to its mux and survives a restart, so the new
/// clock does not wait to hear from a track the previous one measured: it is anchored late
/// enough that the track sent latest keeps the delay when it gets there. A track that has
/// not reached the restart stays on the previous generation while its frames land there and
/// go out before the new generation's first frame; one that would follow it drops as late.
/// as when the skipped groups held a marker the publisher shed. So does a step with no marker
/// at all, as a changeover between two encoders whose clocks disagree, once the delay cannot
/// absorb it: read as drift it would fail [`Buffer::push`], and steering on it would let the
/// step's span of decode time slew the drift in one go. A smaller one is a move in latency.
/// How far ahead of its decode time the source sends each track belongs to its mux and
/// survives a restart, so the new clock does not wait to hear from a track the previous one
/// measured: it is anchored late enough that the track sent latest keeps the delay when it
/// gets there. A track that has not reached the restart stays on the previous generation
/// while its frames land there and go out before the new generation's first frame; one that
/// would follow it drops as late.
/// One crossing the same restart joins the new generation when the frame lands on its
/// clock, neither late nor held more than a delay past everything queued, and opens another
/// otherwise. Before anything has gone out there is nothing to keep in step with, so a
Expand Down Expand Up @@ -694,9 +698,12 @@ struct Floor<K> {
struct Track<T> {
/// Frames in arrival order, each with its generation, decode time and deadline.
queue: VecDeque<(u64, Decode, Instant, T)>,
/// The source's restart and skip counters at the last frame.
/// The source's restart and skip counters at the last frame, when it decodes, and when it
/// arrived.
restart: u64,
skip: u64,
decode: Decode,
arrived: Instant,
/// The generation of the last frame.
generation: u64,
/// Frames are dropped until the next sync frame.
Expand Down Expand Up @@ -822,19 +829,36 @@ impl<K: Ord + Clone, T> Engine<K, T> {

/// Whether a frame moves its track off the clock: a restart, unless the track is crossing
/// one another track already opened a generation for and the frame lands on its clock, or
/// a frame held past [`Self::bound`] after a skip, or while a generation is acquired. The
/// latter is a track reaching the new timeline without a marker of its own, which belongs
/// to that generation, not far ahead on the old clock.
/// a frame held past [`Self::bound`] after a skip, or while a generation is acquired, or a
/// step in its timeline with no marker that the delay cannot absorb. The second is a track
/// reaching the new timeline without a marker of its own, which belongs to that generation,
/// not far ahead on the old clock.
///
/// An unmarked step is told apart from drift and queueing by size. Forward, a frame is held
/// past [`Self::bound`] and reached us more than a delay earlier against its decode time
/// than its track's last: no clock within [`MAX_DRIFT`] moves it that far in the time
/// between them, and frames reach us in the order they are sent, so neither a backlog
/// draining nor a track whose frames are sparse does. Back, a frame decodes before its
/// track's last and is late: a stall only delays what comes next. It is a step on the
/// clock's own generation, since a track still on an earlier one is on that timeline.
/// Held for the next generation, the frame never steers the old clock. Without a delay
/// nothing is held or late, so a step costs nothing there.
fn jumps(&self, key: &K, arrival: &Arrival<T>) -> bool {
let (Some(clock), Some(track)) = (self.clock, self.tracks.get(key)) else {
return false;
};
let deadline = self.deadline(&clock, arrival.decode);
let restarted = track.restart != arrival.restart
&& (track.generation == clock.generation || !self.lands(key, deadline, arrival.arrived));
let leapt = (track.skip != arrival.skip || self.acquire.is_some())
&& deadline.is_some_and(|deadline| deadline > self.bound(key, arrival.arrived));
restarted || leapt
let held = deadline.is_some_and(|deadline| deadline > self.bound(key, arrival.arrived));
let leapt = (track.skip != arrival.skip || self.acquire.is_some()) && held;
let unmarked = !self.delay.is_zero() && track.generation == clock.generation;
let elapsed = arrival.decode.as_nanos() - track.decode.as_nanos();
let ahead = elapsed - since(track.arrived, arrival.arrived);
let drift = (elapsed.abs() as f64 * MAX_DRIFT) as i128;
let forward = held && ahead > self.delay.as_nanos() as i128 + drift;
let back = elapsed < 0 && deadline.is_none_or(|deadline| deadline < arrival.arrived);
restarted || leapt || (unmarked && (forward || back))
}

/// Hold a frame for the generation being acquired, starting one if none is, and anchor it
Expand Down Expand Up @@ -928,13 +952,17 @@ impl<K: Ord + Clone, T> Engine<K, T> {
queue: VecDeque::new(),
restart: arrival.restart,
skip: arrival.skip,
decode: arrival.decode,
arrived: arrival.arrived,
generation,
waiting: false,
resumed: false,
started: false,
});
track.restart = arrival.restart;
track.skip = arrival.skip;
track.decode = arrival.decode;
track.arrived = arrival.arrived;
track.generation = generation;
let push = match deadline {
_ if stale => {
Expand Down Expand Up @@ -2682,6 +2710,103 @@ mod tests {
}
}

/// What [`unmarked`] saw.
struct Stepped {
err: Option<anyhow::Error>,
dropped: u64,
/// Each frame's generation, in the order they went out.
out: Vec<u64>,
/// The clock's drift just before the step, and at the end.
before: f64,
after: f64,
}

/// Feed `secs` of 10 fps frames of `shape`'s tracks through its buffer from a source on our
/// clock, each sent as it decodes with a few ms of jitter that changes every steering step,
/// whose timeline steps by `step` seconds at `at` s with no restart or skip marker.
async fn unmarked(shape: &Shape, at: f64, step: f64, secs: f64) -> Stepped {
// Where the timeline starts, so a step back stays on it.
const BASE: f64 = 20_000.0;
let start = moq_clock::now();
let clock = moq_clock::Deadlines::new(start);
let _entered = clock.enter();
let mut buffer = Buffer::new(shape.delay);
buffer.expect(shape.tracks.iter().copied());
let mut stepped = Stepped {
err: None,
dropped: 0,
out: Vec::new(),
before: 0.0,
after: 0.0,
};
for k in 0..(secs * 10.0) as u64 {
let sent = k as f64 / 10.0;
let jitter = ((k / 20) * 3) % 5;
let arrived = start + Duration::from_secs_f64(sent + jitter as f64 * 0.002);
clock.advance(arrived.max(clock.now()));
if sent < at {
stepped.before = buffer.clock.map_or(0.0, |clock| clock.drift);
}
let decode = BASE + sent + if sent >= at { step } else { 0.0 };
for &track in shape.tracks {
let frame = Arrival {
arrived,
read: arrived,
decode: Timestamp::from_micros((decode * 1e6).round() as u64).unwrap().into(),
restart: 0,
skip: 0,
sync: true,
leading: false,
item: "v",
};
if let Err(err) = buffer.push(track, frame) {
stepped.err = Some(err);
return stepped;
}
}
stepped
.out
.extend(released(&mut buffer).into_iter().map(|(_, generation)| generation));
}
stepped.dropped = buffer.dropped();
stepped.after = buffer.clock.map_or(0.0, |clock| clock.drift);
stepped
}

/// A timeline that steps with no marker, as a changeover between two encoders whose clocks
/// started hours apart, by more than the delay absorbs, either way, before the rate window
/// fills or once it has, re-acquires the clock instead of reading the step as drift: one
/// new generation, nothing dropped, every frame out, and the step never swings the drift
/// further than the slew limit allows over the time that passed.
#[tokio::test(start_paused = true)]
async fn an_unmarked_step_reacquires_the_clock() {
for at in [30.0, 1_200.0] {
for step in [16_844.5, -16_844.5, 2.0, -2.0] {
let secs = at + 60.0;
let run = unmarked(&PAIR, at, step, secs).await;
let case = format!("{step:+} s step at {at} s");
assert!(run.err.is_none(), "{case}: {:?}", run.err);
assert_eq!(run.dropped, 0, "{case}");
assert!(run.out.is_sorted(), "{case}: a generation went out after the next");
assert_eq!(run.out.last(), Some(&1), "{case}: no single new generation");
// What is still held at the end goes out within a delay.
let held = (PAIR.delay.as_secs_f64() * 10.0).ceil() as usize + 1;
let pushed = (secs * 10.0) as usize * PAIR.tracks.len();
assert!(
run.out.len() + held * PAIR.tracks.len() >= pushed,
"{case}: {} of {pushed} went out",
run.out.len()
);
let swing = (run.after - run.before).abs();
assert!(
swing <= MAX_SLEW * (secs - at) * (1.0 + 1e-9),
"{case}: the drift swung {} ppm",
swing * 1e6
);
}
}
}

/// A source within what the clock may follow never fails, though its first frames reached
/// us late, so the clock was anchored that much late, and each frame jitters a few ms, so
/// the first floors, a few ms apart over two seconds, measure a rate far past the limit.
Expand Down
Loading