diff --git a/js/hang/src/container/consumer.holes.test.ts b/js/hang/src/container/consumer.holes.test.ts index bb7843873bb..7f5dc747082 100644 --- a/js/hang/src/container/consumer.holes.test.ts +++ b/js/hang/src/container/consumer.holes.test.ts @@ -442,11 +442,27 @@ test("a hole is waited for the wait while an arrived group plays up to the max d await clock.advance(Duration.millis(1)); expect(reads()).toEqual([0, 500, "gap", 520]); - // Or at once, when the newest content is already the wait past where the group after it begins. + // Content buffered past the wait doesn't skip it early: a burst may still carry the group. write(5n, 700); write(6n, 800); await flush(); + expect(reads()).toEqual([0, 500, "gap", 520]); + await clock.advance(Duration.millis(100)); expect(reads()).toEqual([0, 500, "gap", 520, "gap", 700, 800]); + + // It skips at once when the newest content is already the max delay past the group after it. + write(8n, 900); + write(9n, 1900); + await flush(); + expect(reads()).toEqual([0, 500, "gap", 520, "gap", 700, 800, "gap", 900, 1900]); + + // A burst past the wait that carries the missing group reads with no gap. + write(11n, 2000); + write(12n, 2200); + await flush(); + write(10n, 1950); + await flush(); + expect(reads()).toEqual([0, 500, "gap", 520, "gap", 700, 800, "gap", 900, 1900, 1950, 2000, 2200]); } finally { consumer.close(); track.finish(); diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 87efbe4db7e..0114eebf5b9 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -33,9 +33,9 @@ export interface ConsumerProps { delay?: GetterInit; /** * How long a missing group is waited for before the consumer skips past it, at most `delay.max` - * (default: `delay.max`). It expires once where the first group past it begins falls this far - * behind the newest content, or this much monotonic time after that group arrived, even when - * it is a marker on a paused track. Groups that did arrive still play until they age past + * (default: `delay.max`). It expires this much monotonic time after the first group past it + * arrived, even when that group is a marker on a paused track, or sooner once where that group + * begins falls `delay.max` behind the newest content. Groups that did arrive still play until they age past * `delay.max`, so a player can wait on holes for its playout delay alone while still receiving * the late media it catches up on. */ @@ -855,10 +855,10 @@ export class Consumer { // Otherwise promote #active when the first buffered group continues the timeline we left off at, // when a completed empty group can be walked (empty groups mean nothing), or when the hole // is proven: everything a missing group could still present ends where the first group past - // it begins, so once that start falls the whole wait behind the newest content, all of it - // would arrive too late to play. Mirrors Rust's `aged`. Where presentation left off proves only - // the oldest of it stale, and groups go out newest first, so a flush's newest group lands just - // ahead of the groups behind it, which would then arrive below the cursor and never play. + // it begins, so once that start falls the max delay behind the newest content, all of it + // would arrive too late to play. Mirrors Rust's `aged`. Where presentation left off proves + // only the oldest of it stale, and groups go out newest first, so a flush's newest group lands + // just ahead of the groups behind it, which would then arrive below the cursor and never play. // Proving it here matters: #checkMaxDelay would instead drop the head, the very group to play // next. // After track termination no missing group can arrive, so drain across any remaining gap. @@ -877,7 +877,10 @@ export class Consumer { const wait = this.#holeWait(this.#delay.peek(), this.#wait.peek()); const bound = this.#groups.find((group) => group.start !== undefined)?.start; const edge = this.#liveEdge(); - const aged = bound !== undefined && edge !== undefined && edge - bound >= wait; + // Measured against the max delay, not the wait: a reader starved of CPU takes in a burst of + // groups at once, and the missing one may land in the same burst. + const aged = + bound !== undefined && edge !== undefined && edge - bound >= maxMicros(this.#delay.peek()); const elapsed = now().since(this.#hole).asMicros() >= wait; // The ranges ask for nothing in between, so nothing there is owed. const next = this.#track.nextInRange(this.#active); diff --git a/quest/m0/test-flakes-2/README.md b/quest/m0/test-flakes-2/README.md index 56235316edb..9fb69a97173 100644 --- a/quest/m0/test-flakes-2/README.md +++ b/quest/m0/test-flakes-2/README.md @@ -1,4 +1,4 @@ -# More tests hold up under load +# [S] More tests hold up under load ## Goal diff --git a/quest/m0/test-flakes-2/interop-python-cpp-audio-gap.md b/quest/m0/test-flakes-2/interop-python-cpp-audio-gap.md deleted file mode 100644 index b3b754ad6f6..00000000000 --- a/quest/m0/test-flakes-2/interop-python-cpp-audio-gap.md +++ /dev/null @@ -1,22 +0,0 @@ -# [S] Interop's python -> cpp cell holds up under load - -## Goal - -`just test interop --all` passes the `python -> cpp` media cell on a loaded -host without it ending in `error: audio gap`. - -## Plan - -Planned 2026-11-13 under the repository's flaky-test rule, while iterating -[moq-dev/moq#8599](https://github.com/moq-dev/moq/pull/8599). One of three -`just test interop --all --publishers go,rust,python --subscribers cpp,js,js-native-node,rust` -runs on a shared, loaded host failed `python -> cpp` about 3 seconds in, the -cpp subscriber reporting `error: audio gap`; the other two runs passed, and -the logs did not show the grant close #8587 fixed. The failing run's logs are -gone. - -Reproduce with `just test interop --publishers python --subscribers cpp` in a -loop under load, keeping the logs, and find whether the Python publisher -skips audio timestamps (a stalled encoder or a dropped group) or the cpp -subscriber's gap check misreads a legitimate group boundary; fix whichever -end is wrong, rather than widening the gap tolerance. diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 460e0025466..89ed4781ca7 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -66,11 +66,12 @@ pub(crate) enum Item { /// [`Delay::max`] measures. With the default max delay of zero, /// the consumer skips aggressively: any group that has a newer alternative is dropped. /// With a non-zero max delay, slow groups are tolerated up to that budget before being -/// skipped. A missing sequence gets the same tolerance, or the shorter -/// [`set_wait`](Self::set_wait): there is no way to tell a stream that lost the delivery race -/// from one the cache evicted. Its wait also expires that much monotonic time after the first -/// group past it arrived, so a marker behind a hole is delivered even when the track is paused, -/// and the holes a stalled source left behind expire together rather than one wait each. +/// skipped. A missing sequence gets the same tolerance: there is no way to tell a stream that +/// lost the delivery race from one the cache evicted, and a reader starved of CPU may take the +/// missing group in the same burst as the ones past it. Its wait, [`set_wait`](Self::set_wait) +/// or the max delay, also expires that much monotonic time after the first group past it +/// arrived, so a marker behind a hole is delivered even when the track is paused, and the holes +/// a stalled source left behind expire together rather than one wait each. /// /// Delivery starts at the floor the /// [`Subscription::ranges`](moq_net::track::Subscription::ranges) name, raised to where the @@ -731,9 +732,11 @@ impl Consumer { // Walk the cursor over missing sequences below the first arrived group. // Groups race on independent QUIC streams (newer ones at higher priority), // so a buffered higher sequence proves nothing: the missing one may be - // merely late, and there is no way to tell that from an eviction. The age - // budget is the gate: everything a missing group could still present is - // bounded by where the next stamped group begins. A finished track closes + // merely late, and there is no way to tell that from an eviction. The wait, + // in wall time since a later group arrived, is the gate. Media buffered past + // the hole proves it stale only past the max delay, not the shorter wait: a + // reader starved of CPU takes in a burst of groups at once, and the missing + // one may land in the same burst. A finished track closes // the gap outright, since no new group can arrive. That proof covers only // sequences that never arrived: finishing the track ends new groups, not // the frames still flowing on ones already open, so arrived groups are @@ -750,7 +753,7 @@ impl Consumer { let since = front.late; let wait = self.wait.map_or(self.max_delay, |wait| wait.min(self.max_delay)); let elapsed = since.elapsed() >= wait; - let aged = next_group.is_some_and(|(_, start)| max_timestamp.saturating_sub(start) >= wait); + let aged = next_group.is_some_and(|(_, start)| max_timestamp.saturating_sub(start) >= self.max_delay); let excluded = match &self.reader { Reader::Reading(track) | Reader::Failed(track, _) => { track.next_in_range(self.current).is_none_or(|next| next >= sequence) @@ -770,6 +773,16 @@ impl Consumer { if !empty && next_start.is_none_or(|start| !pts_contiguous(self.presented_end, start)) { self.lose(); } + tracing::debug!( + old = self.current, + new = sequence, + finished, + elapsed, + aged, + excluded, + full, + "skipping missing groups" + ); self.current = sequence; self.note_group_edge(); continue; @@ -1210,7 +1223,9 @@ impl Consumer { } } - /// Wait at most `wait` for a missing group before skipping past it (default: the max delay). + /// Wait at most `wait` of monotonic time, after the first group past a missing one arrives, + /// before skipping past it (default: the max delay). Content age skips it once that group + /// begins the max delay behind the newest content, whatever the wait. /// /// It never exceeds [`Delay::max`]. Groups that did arrive still play until they age past the /// max delay, so a player can wait on holes for its playout delay alone while still receiving @@ -1957,6 +1972,36 @@ mod tests { assert_eq!(read, ["gap", "frame 2", "gap", "frame 4", "gap", "frame 6"]); } + /// A burst of later groups spanning past the wait doesn't age out a missing group that lands in + /// the same burst: a reader starved of CPU takes in every stream at once. + #[tokio::test(start_paused = true)] + async fn a_burst_past_the_wait_does_not_age_out_a_hole() { + let mut track = track_producer("data", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscriber = track.subscribe( + moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1))), + ); + let mut reader = Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Data)); + reader.set_wait(Duration::from_millis(100)); + write_group(&mut track, 0, &[ts(0)]); + assert!(matches!( + reader.read().await.unwrap(), + Some(Read::Frame { group: 0, .. }) + )); + write_group(&mut track, 2, &[ts(40_000), ts(200_000)]); + write_group(&mut track, 3, &[ts(300_000)]); + assert!(reader.poll_read(&kio::Waiter::noop()).is_pending()); + write_group(&mut track, 1, &[ts(20_000)]); + let mut read = Vec::new(); + while let Poll::Ready(Ok(Some(item))) = reader.poll_read(&kio::Waiter::noop()) { + read.push(match item { + Read::Frame { group, .. } => format!("frame {group}"), + Read::Gap { .. } => "gap".to_string(), + Read::End { .. } => "end".to_string(), + }); + } + assert_eq!(read, ["frame 1", "frame 2", "frame 2", "frame 3"]); + } + /// A group that arrives late below an earlier one inherits that earlier arrival: the holes /// around it were already observable then. #[tokio::test(start_paused = true)] @@ -2034,11 +2079,17 @@ mod tests { tokio::time::advance(Duration::from_millis(1)).await; assert_eq!(drain(&mut reader), ["gap", "frame 3"]); - // Or at once, when the newest content is already the wait past where the group after it - // begins. + // Content buffered past the wait doesn't skip it early: a burst may still carry the group. write_group(&mut track, 5, &[ts(700_000)]); write_group(&mut track, 6, &[ts(800_000)]); + assert!(drain(&mut reader).is_empty()); + tokio::time::advance(Duration::from_millis(100)).await; assert_eq!(drain(&mut reader), ["gap", "frame 5", "frame 6"]); + + // It skips at once when the newest content is already the max delay past the group after it. + write_group(&mut track, 8, &[ts(900_000)]); + write_group(&mut track, 9, &[ts(1_900_000)]); + assert_eq!(drain(&mut reader), ["gap", "frame 8", "frame 9"]); } /// A malformed data marker group found while looking ahead fails the reader, rather than diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json index bdb037f3577..dce4c96be65 100644 --- a/test/audio-quality/budgets.json +++ b/test/audio-quality/budgets.json @@ -432,10 +432,10 @@ "underrun_ms_per_min": 0, "trimmed_ms_per_min": 0, "stalled_share": 0.005, - "silence_share": 0.009, + "silence_share": 0.007, "target_ms_p95": 343.2, "av_lag_ms_p50": 11, - "av_lag_ms_p95": 104.8 + "av_lag_ms_p95": 102.7 }, { "runtime": "replay", @@ -466,8 +466,8 @@ "stalled_share": 0, "silence_share": 0, "target_ms_p95": 223.2, - "av_lag_ms_p50": -4.2, - "av_lag_ms_p95": 1.4 + "av_lag_ms_p50": -2, + "av_lag_ms_p95": 3.6 }, { "runtime": "replay", diff --git a/test/interop/clients/c/subscribe.c b/test/interop/clients/c/subscribe.c index 82b4d6a210a..0c91c92a811 100644 --- a/test/interop/clients/c/subscribe.c +++ b/test/interop/clients/c/subscribe.c @@ -259,6 +259,12 @@ static void start_audio(ctx_t *c, int32_t catalog) { moq_audio_decoder_output out; memset(&out, 0, sizeof(out)); out.format = MOQ_AUDIO_SAMPLE_FORMAT_F32; + // moq-audio waits on a missing group for its playout delay, measured from arrival timing + // at tens of ms, so a runtime starved longer than that skips a group that is only late. + // This reader checks delivery between implementations, not a real-time deadline: it holds + // a playout delay of the max delay, waiting on a hole for the same budget it grants a late + // group. + out.delay.min_us = MAX_DELAY_US; out.delay.max_us = MAX_DELAY_US; int32_t track = moq_decode_audio((uint32_t)catalog, i, &out, on_audio, &c->audio[i]); pthread_mutex_lock(&c->mu); diff --git a/test/interop/clients/cpp/main.cpp b/test/interop/clients/cpp/main.cpp index f31db1e6a4f..536077fbc8d 100644 --- a/test/interop/clients/cpp/main.cpp +++ b/test/interop/clients/cpp/main.cpp @@ -310,7 +310,11 @@ void decode_audio( } moq::AudioDecoderOutput output{}; output.format = moq::AudioSampleFormat::kF32; - output.delay = moq::Delay{std::chrono::nanoseconds(0), MAX_DELAY}; + // moq-audio waits on a missing group for its playout delay, measured from arrival timing at + // tens of ms, so a runtime starved longer than that skips a group that is only late. This + // reader checks delivery between implementations, not a real-time deadline: it holds a playout + // delay of the max delay, waiting on a hole for the same budget it grants a late group. + output.delay = moq::Delay{MAX_DELAY, MAX_DELAY}; auto decoder = until(moq::AudioConsumer::subscribe(consumer, id, audio, output), deadline, "audio consumer"); std::ofstream log(out + ".txt"); @@ -324,8 +328,13 @@ void decode_audio( fail("audio track ended early"); } // moq-audio reports a hole it saw directly; tone.py checks the timestamps too. - if (std::holds_alternative(read->get_variant())) { - fail("audio gap"); + if (const auto *gap = std::get_if(&read->get_variant())) { + const auto at = gap->timestamp.value / gap->timestamp.scale * 1'000'000 + + gap->timestamp.value % gap->timestamp.scale * 1'000'000 / gap->timestamp.scale; + const auto us = std::chrono::duration_cast(gap->duration).count(); + fail(("audio gap after " + std::to_string(decoded) + " samples, at " + std::to_string(at) + "us for " + + std::to_string(us) + "us") + .c_str()); } const auto *variant = std::get_if(&read->get_variant()); if (!variant) { diff --git a/test/interop/clients/go/main.go b/test/interop/clients/go/main.go index b8a34ae6b73..32f08c6ad24 100644 --- a/test/interop/clients/go/main.go +++ b/test/interop/clients/go/main.go @@ -469,7 +469,11 @@ func decodeAudio(ctx context.Context, consumer *moq.BroadcastConsumer, id string out += "." + codec decoder, err := moqaudio.Subscribe(ctx, consumer, id, rendition, moqaudio.DecoderOutput{ Format: moq.AudioSampleFormatF32, - Delay: moq.Delay{Max: maxDelay}, + // moq-audio waits on a missing group for its playout delay, measured from arrival timing at tens + // of ms, so a runtime starved longer than that skips a group that is only late. This reader + // checks delivery between implementations, not a real-time deadline: it holds a playout delay of + // the max delay, waiting on a hole for the same budget it grants a late group. + Delay: moq.Delay{Min: maxDelay, Max: maxDelay}, }) if err != nil { return err @@ -501,7 +505,7 @@ func decodeAudio(ctx context.Context, consumer *moq.BroadcastConsumer, id string frame = read.Frame // moq-audio reports a hole it saw directly; tone.py checks the timestamps too. case moqaudio.ReadGap: - return fmt.Errorf("audio gap after %d samples", decoded) + return fmt.Errorf("audio gap after %d samples, at %v for %v", decoded, read.Timestamp, read.Duration) default: // nil, the track's end, or a ReadEnd, the encoder's return errors.New("audio track ended early") } diff --git a/test/interop/clients/python/interop.py b/test/interop/clients/python/interop.py index 1af2c2695bd..f757e1f60a4 100644 --- a/test/interop/clients/python/interop.py +++ b/test/interop/clients/python/interop.py @@ -263,11 +263,15 @@ async def _decode_audio( exclusively, so two renditions of one codec fail rather than overwrite each other. """ out = f"{out}.{audio.codec.split('.')[0]}" + # moq-audio waits on a missing group for its playout delay, measured from arrival timing at + # tens of ms, so a runtime starved longer than that skips a group that is only late. This + # reader checks delivery between implementations, not a real-time deadline: it holds a playout + # delay of the max delay, waiting on a hole for the same budget it grants a late group. decoder = await moq.audio.Consumer.subscribe( consumer, id, audio, - moq.audio.DecoderOutput(format=moq.audio.SampleFormat.F32, delay=moq.Delay(max=MAX_DELAY)), + moq.audio.DecoderOutput(format=moq.audio.SampleFormat.F32, delay=moq.Delay(min=MAX_DELAY, max=MAX_DELAY)), ) frame_bytes = 4 * audio.channel_count decoded = 0 @@ -276,7 +280,7 @@ async def _decode_audio( async for frame in decoder: # moq-audio reports a hole it saw directly; tone.py checks the timestamps too. if isinstance(frame, moq.audio.Gap): - raise RuntimeError(f"audio gap after {decoded} samples") + raise RuntimeError(f"audio gap after {decoded} samples, at {frame.timestamp} for {frame.duration}") if isinstance(frame, moq.audio.End): raise RuntimeError("audio track ended early") samples = len(frame.data) // frame_bytes diff --git a/test/interop/clients/rust/audio.rs b/test/interop/clients/rust/audio.rs index 9ada6204099..915bbd30444 100644 --- a/test/interop/clients/rust/audio.rs +++ b/test/interop/clients/rust/audio.rs @@ -100,7 +100,12 @@ async fn decode( // runtime starved by the others can fall a second behind. That is this reader's speed, not the // delivery under test. A track's max-age caps it, so interop.sh's Rust publisher keeps its // media at least this long. + // moq-audio waits on a missing group for its playout delay, measured from arrival timing at tens + // of ms, so a runtime starved longer than that skips a group that is only late. This reader + // checks delivery between implementations, not a real-time deadline: it holds a playout delay of + // the max delay, waiting on a hole for the same budget it grants a late group. options.delay.max = Some(Duration::from_secs(5)); + options.delay.min = options.delay.max; let mut decoder = moq_audio::decode::Consumer::new(&rendition, audio, name, options).await?; let rate = decoder.sample_rate(); let channels = decoder.layout().channels(); @@ -125,7 +130,10 @@ async fn decode( let frame = match read { moq_audio::decode::Read::Frame(frame) => frame, // moq-audio reports a hole it saw directly; tone.py checks the timestamps too. - moq_audio::decode::Read::Gap { .. } => anyhow::bail!("{name}: audio gap after {decoded} samples"), + moq_audio::decode::Read::Gap { timestamp, duration } => anyhow::bail!( + "{name}: audio gap after {decoded} samples, at {}us for {duration:?}", + timestamp.as_micros() + ), moq_audio::decode::Read::End { .. } => anyhow::bail!("{name}: audio track ended early"), }; let samples = frame.data.len() as u64 / (4 * channels as u64);