Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion js/hang/src/container/consumer.holes.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -442,11 +442,27 @@ test("a hole is waited for the wait while an arrived group plays up to the max d
await clock.advance(Duration.millis(1));
expect(reads()).toEqual([0, 500, "gap", 520]);

// Or at once, when the newest content is already the wait past where the group after it begins.
// Content buffered past the wait doesn't skip it early: a burst may still carry the group.
write(5n, 700);
write(6n, 800);
await flush();
expect(reads()).toEqual([0, 500, "gap", 520]);
await clock.advance(Duration.millis(100));
expect(reads()).toEqual([0, 500, "gap", 520, "gap", 700, 800]);

// It skips at once when the newest content is already the max delay past the group after it.
write(8n, 900);
write(9n, 1900);
await flush();
expect(reads()).toEqual([0, 500, "gap", 520, "gap", 700, 800, "gap", 900, 1900]);

// A burst past the wait that carries the missing group reads with no gap.
write(11n, 2000);
write(12n, 2200);
await flush();
write(10n, 1950);
await flush();
expect(reads()).toEqual([0, 500, "gap", 520, "gap", 700, 800, "gap", 900, 1900, 1950, 2000, 2200]);
} finally {
consumer.close();
track.finish();
Expand Down
19 changes: 11 additions & 8 deletions js/hang/src/container/consumer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ export interface ConsumerProps {
delay?: GetterInit<Moq.Track.Delay>;
/**
* How long a missing group is waited for before the consumer skips past it, at most `delay.max`
* (default: `delay.max`). It expires once where the first group past it begins falls this far
* behind the newest content, or this much monotonic time after that group arrived, even when
* it is a marker on a paused track. Groups that did arrive still play until they age past
* (default: `delay.max`). It expires this much monotonic time after the first group past it
* arrived, even when that group is a marker on a paused track, or sooner once where that group
* begins falls `delay.max` behind the newest content. Groups that did arrive still play until they age past
* `delay.max`, so a player can wait on holes for its playout delay alone while still receiving
* the late media it catches up on.
*/
Expand Down Expand Up @@ -855,10 +855,10 @@ export class Consumer {
// Otherwise promote #active when the first buffered group continues the timeline we left off at,
// when a completed empty group can be walked (empty groups mean nothing), or when the hole
// is proven: everything a missing group could still present ends where the first group past
// it begins, so once that start falls the whole wait behind the newest content, all of it
// would arrive too late to play. Mirrors Rust's `aged`. Where presentation left off proves only
// the oldest of it stale, and groups go out newest first, so a flush's newest group lands just
// ahead of the groups behind it, which would then arrive below the cursor and never play.
// it begins, so once that start falls the max delay behind the newest content, all of it
// would arrive too late to play. Mirrors Rust's `aged`. Where presentation left off proves
// only the oldest of it stale, and groups go out newest first, so a flush's newest group lands
// just ahead of the groups behind it, which would then arrive below the cursor and never play.
// Proving it here matters: #checkMaxDelay would instead drop the head, the very group to play
// next.
// After track termination no missing group can arrive, so drain across any remaining gap.
Expand All @@ -877,7 +877,10 @@ export class Consumer {
const wait = this.#holeWait(this.#delay.peek(), this.#wait.peek());
const bound = this.#groups.find((group) => group.start !== undefined)?.start;
const edge = this.#liveEdge();
const aged = bound !== undefined && edge !== undefined && edge - bound >= wait;
// Measured against the max delay, not the wait: a reader starved of CPU takes in a burst of
// groups at once, and the missing one may land in the same burst.
const aged =
bound !== undefined && edge !== undefined && edge - bound >= maxMicros(this.#delay.peek());
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
const elapsed = now().since(this.#hole).asMicros() >= wait;
// The ranges ask for nothing in between, so nothing there is owed.
const next = this.#track.nextInRange(this.#active);
Expand Down
2 changes: 1 addition & 1 deletion quest/m0/test-flakes-2/README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# More tests hold up under load
# [S] More tests hold up under load

## Goal

Expand Down
22 changes: 0 additions & 22 deletions quest/m0/test-flakes-2/interop-python-cpp-audio-gap.md

This file was deleted.

75 changes: 63 additions & 12 deletions rs/moq-mux/src/container/consumer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,12 @@ pub(crate) enum Item {
/// [`Delay::max`] measures. With the default max delay of zero,
/// the consumer skips aggressively: any group that has a newer alternative is dropped.
/// With a non-zero max delay, slow groups are tolerated up to that budget before being
/// skipped. A missing sequence gets the same tolerance, or the shorter
/// [`set_wait`](Self::set_wait): there is no way to tell a stream that lost the delivery race
/// from one the cache evicted. Its wait also expires that much monotonic time after the first
/// group past it arrived, so a marker behind a hole is delivered even when the track is paused,
/// and the holes a stalled source left behind expire together rather than one wait each.
/// skipped. A missing sequence gets the same tolerance: there is no way to tell a stream that
/// lost the delivery race from one the cache evicted, and a reader starved of CPU may take the
/// missing group in the same burst as the ones past it. Its wait, [`set_wait`](Self::set_wait)
/// or the max delay, also expires that much monotonic time after the first group past it
/// arrived, so a marker behind a hole is delivered even when the track is paused, and the holes
/// a stalled source left behind expire together rather than one wait each.
///
/// Delivery starts at the floor the
/// [`Subscription::ranges`](moq_net::track::Subscription::ranges) name, raised to where the
Expand Down Expand Up @@ -731,9 +732,11 @@ impl<F: Container> Consumer<F> {
// Walk the cursor over missing sequences below the first arrived group.
// Groups race on independent QUIC streams (newer ones at higher priority),
// so a buffered higher sequence proves nothing: the missing one may be
// merely late, and there is no way to tell that from an eviction. The age
// budget is the gate: everything a missing group could still present is
// bounded by where the next stamped group begins. A finished track closes
// merely late, and there is no way to tell that from an eviction. The wait,
// in wall time since a later group arrived, is the gate. Media buffered past
// the hole proves it stale only past the max delay, not the shorter wait: a
// reader starved of CPU takes in a burst of groups at once, and the missing
// one may land in the same burst. A finished track closes
// the gap outright, since no new group can arrive. That proof covers only
// sequences that never arrived: finishing the track ends new groups, not
// the frames still flowing on ones already open, so arrived groups are
Expand All @@ -750,7 +753,7 @@ impl<F: Container> Consumer<F> {
let since = front.late;
let wait = self.wait.map_or(self.max_delay, |wait| wait.min(self.max_delay));
let elapsed = since.elapsed() >= wait;
let aged = next_group.is_some_and(|(_, start)| max_timestamp.saturating_sub(start) >= wait);
let aged = next_group.is_some_and(|(_, start)| max_timestamp.saturating_sub(start) >= self.max_delay);
let excluded = match &self.reader {
Reader::Reading(track) | Reader::Failed(track, _) => {
track.next_in_range(self.current).is_none_or(|next| next >= sequence)
Expand All @@ -770,6 +773,16 @@ impl<F: Container> Consumer<F> {
if !empty && next_start.is_none_or(|start| !pts_contiguous(self.presented_end, start)) {
self.lose();
}
tracing::debug!(
old = self.current,
new = sequence,
finished,
elapsed,
aged,
excluded,
full,
"skipping missing groups"
);
self.current = sequence;
self.note_group_edge();
continue;
Expand Down Expand Up @@ -1210,7 +1223,9 @@ impl<F: Container> Consumer<F> {
}
}

/// Wait at most `wait` for a missing group before skipping past it (default: the max delay).
/// Wait at most `wait` of monotonic time, after the first group past a missing one arrives,
/// before skipping past it (default: the max delay). Content age skips it once that group
/// begins the max delay behind the newest content, whatever the wait.
///
/// It never exceeds [`Delay::max`]. Groups that did arrive still play until they age past the
/// max delay, so a player can wait on holes for its playout delay alone while still receiving
Expand Down Expand Up @@ -1957,6 +1972,36 @@ mod tests {
assert_eq!(read, ["gap", "frame 2", "gap", "frame 4", "gap", "frame 6"]);
}

/// A burst of later groups spanning past the wait doesn't age out a missing group that lands in
/// the same burst: a reader starved of CPU takes in every stream at once.
#[tokio::test(start_paused = true)]
async fn a_burst_past_the_wait_does_not_age_out_a_hole() {
let mut track = track_producer("data", hang::container::track_info(hang::catalog::PRIORITY.audio));
let subscriber = track.subscribe(
moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1))),
);
let mut reader = Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Data));
reader.set_wait(Duration::from_millis(100));
write_group(&mut track, 0, &[ts(0)]);
assert!(matches!(
reader.read().await.unwrap(),
Some(Read::Frame { group: 0, .. })
));
write_group(&mut track, 2, &[ts(40_000), ts(200_000)]);
write_group(&mut track, 3, &[ts(300_000)]);
assert!(reader.poll_read(&kio::Waiter::noop()).is_pending());
write_group(&mut track, 1, &[ts(20_000)]);
let mut read = Vec::new();
while let Poll::Ready(Ok(Some(item))) = reader.poll_read(&kio::Waiter::noop()) {
read.push(match item {
Read::Frame { group, .. } => format!("frame {group}"),
Read::Gap { .. } => "gap".to_string(),
Read::End { .. } => "end".to_string(),
});
}
assert_eq!(read, ["frame 1", "frame 2", "frame 2", "frame 3"]);
}

/// A group that arrives late below an earlier one inherits that earlier arrival: the holes
/// around it were already observable then.
#[tokio::test(start_paused = true)]
Expand Down Expand Up @@ -2034,11 +2079,17 @@ mod tests {
tokio::time::advance(Duration::from_millis(1)).await;
assert_eq!(drain(&mut reader), ["gap", "frame 3"]);

// Or at once, when the newest content is already the wait past where the group after it
// begins.
// Content buffered past the wait doesn't skip it early: a burst may still carry the group.
write_group(&mut track, 5, &[ts(700_000)]);
write_group(&mut track, 6, &[ts(800_000)]);
assert!(drain(&mut reader).is_empty());
tokio::time::advance(Duration::from_millis(100)).await;
assert_eq!(drain(&mut reader), ["gap", "frame 5", "frame 6"]);

// It skips at once when the newest content is already the max delay past the group after it.
write_group(&mut track, 8, &[ts(900_000)]);
write_group(&mut track, 9, &[ts(1_900_000)]);
assert_eq!(drain(&mut reader), ["gap", "frame 8", "frame 9"]);
}

/// A malformed data marker group found while looking ahead fails the reader, rather than
Expand Down
8 changes: 4 additions & 4 deletions test/audio-quality/budgets.json
Original file line number Diff line number Diff line change
Expand Up @@ -432,10 +432,10 @@
"underrun_ms_per_min": 0,
"trimmed_ms_per_min": 0,
"stalled_share": 0.005,
"silence_share": 0.009,
"silence_share": 0.007,
"target_ms_p95": 343.2,
"av_lag_ms_p50": 11,
"av_lag_ms_p95": 104.8
"av_lag_ms_p95": 102.7
},
{
"runtime": "replay",
Expand Down Expand Up @@ -466,8 +466,8 @@
"stalled_share": 0,
"silence_share": 0,
"target_ms_p95": 223.2,
"av_lag_ms_p50": -4.2,
"av_lag_ms_p95": 1.4
"av_lag_ms_p50": -2,
"av_lag_ms_p95": 3.6
},
{
"runtime": "replay",
Expand Down
6 changes: 6 additions & 0 deletions test/interop/clients/c/subscribe.c
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,12 @@ static void start_audio(ctx_t *c, int32_t catalog) {
moq_audio_decoder_output out;
memset(&out, 0, sizeof(out));
out.format = MOQ_AUDIO_SAMPLE_FORMAT_F32;
// moq-audio waits on a missing group for its playout delay, measured from arrival timing
// at tens of ms, so a runtime starved longer than that skips a group that is only late.
// This reader checks delivery between implementations, not a real-time deadline: it holds
// a playout delay of the max delay, waiting on a hole for the same budget it grants a late
// group.
out.delay.min_us = MAX_DELAY_US;
out.delay.max_us = MAX_DELAY_US;
int32_t track = moq_decode_audio((uint32_t)catalog, i, &out, on_audio, &c->audio[i]);
pthread_mutex_lock(&c->mu);
Expand Down
15 changes: 12 additions & 3 deletions test/interop/clients/cpp/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,11 @@ void decode_audio(
}
moq::AudioDecoderOutput output{};
output.format = moq::AudioSampleFormat::kF32;
output.delay = moq::Delay{std::chrono::nanoseconds(0), MAX_DELAY};
// moq-audio waits on a missing group for its playout delay, measured from arrival timing at
// tens of ms, so a runtime starved longer than that skips a group that is only late. This
// reader checks delivery between implementations, not a real-time deadline: it holds a playout
// delay of the max delay, waiting on a hole for the same budget it grants a late group.
output.delay = moq::Delay{MAX_DELAY, MAX_DELAY};
auto decoder = until(moq::AudioConsumer::subscribe(consumer, id, audio, output), deadline, "audio consumer");

std::ofstream log(out + ".txt");
Expand All @@ -324,8 +328,13 @@ void decode_audio(
fail("audio track ended early");
}
// moq-audio reports a hole it saw directly; tone.py checks the timestamps too.
if (std::holds_alternative<moq::AudioRead::kGap>(read->get_variant())) {
fail("audio gap");
if (const auto *gap = std::get_if<moq::AudioRead::kGap>(&read->get_variant())) {
const auto at = gap->timestamp.value / gap->timestamp.scale * 1'000'000 +
gap->timestamp.value % gap->timestamp.scale * 1'000'000 / gap->timestamp.scale;
const auto us = std::chrono::duration_cast<std::chrono::microseconds>(gap->duration).count();
fail(("audio gap after " + std::to_string(decoded) + " samples, at " + std::to_string(at) + "us for " +
std::to_string(us) + "us")
.c_str());
}
const auto *variant = std::get_if<moq::AudioRead::kFrame>(&read->get_variant());
if (!variant) {
Expand Down
8 changes: 6 additions & 2 deletions test/interop/clients/go/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -469,7 +469,11 @@ func decodeAudio(ctx context.Context, consumer *moq.BroadcastConsumer, id string
out += "." + codec
decoder, err := moqaudio.Subscribe(ctx, consumer, id, rendition, moqaudio.DecoderOutput{
Format: moq.AudioSampleFormatF32,
Delay: moq.Delay{Max: maxDelay},
// moq-audio waits on a missing group for its playout delay, measured from arrival timing at tens
// of ms, so a runtime starved longer than that skips a group that is only late. This reader
// checks delivery between implementations, not a real-time deadline: it holds a playout delay of
// the max delay, waiting on a hole for the same budget it grants a late group.
Delay: moq.Delay{Min: maxDelay, Max: maxDelay},
})
if err != nil {
return err
Expand Down Expand Up @@ -501,7 +505,7 @@ func decodeAudio(ctx context.Context, consumer *moq.BroadcastConsumer, id string
frame = read.Frame
// moq-audio reports a hole it saw directly; tone.py checks the timestamps too.
case moqaudio.ReadGap:
return fmt.Errorf("audio gap after %d samples", decoded)
return fmt.Errorf("audio gap after %d samples, at %v for %v", decoded, read.Timestamp, read.Duration)
default: // nil, the track's end, or a ReadEnd, the encoder's
return errors.New("audio track ended early")
}
Expand Down
8 changes: 6 additions & 2 deletions test/interop/clients/python/interop.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,11 +263,15 @@ async def _decode_audio(
exclusively, so two renditions of one codec fail rather than overwrite each other.
"""
out = f"{out}.{audio.codec.split('.')[0]}"
# moq-audio waits on a missing group for its playout delay, measured from arrival timing at
# tens of ms, so a runtime starved longer than that skips a group that is only late. This
# reader checks delivery between implementations, not a real-time deadline: it holds a playout
# delay of the max delay, waiting on a hole for the same budget it grants a late group.
decoder = await moq.audio.Consumer.subscribe(
consumer,
id,
audio,
moq.audio.DecoderOutput(format=moq.audio.SampleFormat.F32, delay=moq.Delay(max=MAX_DELAY)),
moq.audio.DecoderOutput(format=moq.audio.SampleFormat.F32, delay=moq.Delay(min=MAX_DELAY, max=MAX_DELAY)),
)
frame_bytes = 4 * audio.channel_count
decoded = 0
Expand All @@ -276,7 +280,7 @@ async def _decode_audio(
async for frame in decoder:
# moq-audio reports a hole it saw directly; tone.py checks the timestamps too.
if isinstance(frame, moq.audio.Gap):
raise RuntimeError(f"audio gap after {decoded} samples")
raise RuntimeError(f"audio gap after {decoded} samples, at {frame.timestamp} for {frame.duration}")
if isinstance(frame, moq.audio.End):
raise RuntimeError("audio track ended early")
samples = len(frame.data) // frame_bytes
Expand Down
10 changes: 9 additions & 1 deletion test/interop/clients/rust/audio.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,12 @@ async fn decode(
// runtime starved by the others can fall a second behind. That is this reader's speed, not the
// delivery under test. A track's max-age caps it, so interop.sh's Rust publisher keeps its
// media at least this long.
// moq-audio waits on a missing group for its playout delay, measured from arrival timing at tens
// of ms, so a runtime starved longer than that skips a group that is only late. This reader
// checks delivery between implementations, not a real-time deadline: it holds a playout delay of
// the max delay, waiting on a hole for the same budget it grants a late group.
options.delay.max = Some(Duration::from_secs(5));
options.delay.min = options.delay.max;
let mut decoder = moq_audio::decode::Consumer::new(&rendition, audio, name, options).await?;
let rate = decoder.sample_rate();
let channels = decoder.layout().channels();
Expand All @@ -125,7 +130,10 @@ async fn decode(
let frame = match read {
moq_audio::decode::Read::Frame(frame) => frame,
// moq-audio reports a hole it saw directly; tone.py checks the timestamps too.
moq_audio::decode::Read::Gap { .. } => anyhow::bail!("{name}: audio gap after {decoded} samples"),
moq_audio::decode::Read::Gap { timestamp, duration } => anyhow::bail!(
"{name}: audio gap after {decoded} samples, at {}us for {duration:?}",
timestamp.as_micros()
),
moq_audio::decode::Read::End { .. } => anyhow::bail!("{name}: audio track ended early"),
};
let samples = frame.data.len() as u64 / (4 * channels as u64);
Expand Down