diff --git a/quest/m0/ci/slow-tests.md b/quest/m0/ci/slow-tests.md index cc48b812f62..9a41aa83fbf 100644 --- a/quest/m0/ci/slow-tests.md +++ b/quest/m0/ci/slow-tests.md @@ -2,11 +2,12 @@ ## Goal -moq's slowest 1% of tests (103 tests, 289 of 501 test CPU-seconds in one full run) -take under half that CPU, with no loss of coverage. Before any change, freeze the -cohort by test ID from one full `cargo nextest` run and list it in this quest's PR; -measure that same set after, on the same machine. Per-test CPU is each test process's -user plus system time, not nextest's wall-clock duration. +The 103 test IDs frozen in [moq-dev/moq#6914](https://github.com/moq-dev/moq/pull/6914) +(the slowest `ceil(10,275 / 100)` of one full normal `cargo nextest` run) take under +68.92 CPU-seconds together, half their 137.83 s baseline, with every case and its +coverage preserved. Per-test CPU is the test process's user plus system time, not +nextest's wall-clock duration. Measure the same IDs on the same machine as the baseline +it is compared against; missing or failed cases are not imputed. ## Plan @@ -15,11 +16,62 @@ Decided with the maintainer on 2026-10-26, from kixelated/rust2's of tests make up 57% of what either package selection or a function-level selector would run, so cutting them beats building a selector. -- Start with moq-mux's container import tests (137 s) and moq-net's integration tests - (42 s). - Prefer making a test cheaper: smaller fixtures, shared setup, simulated time instead of real waits, fewer redundant cases. - Moving a test to nightly only follows this epic's rule: evidence and an explicit coverage tradeoff for each. +### Measurement + +[moq-dev/moq#6914](https://github.com/moq-dev/moq/pull/6914) holds the frozen list with +per-test baseline and after CPU, plus the runner wrapper and command. The wrapper is a +`CARGO_TARGET__RUNNER` that records each nextest test child's `wait4` user plus +system time, run under the normal `just rs test --workspace --exclude moq-net-fuzz +--exclude moq-quic-fuzz` with zero retries. On a different machine, re-measure the +baseline source (`6f6a3ae21432a3df74d4c432451f3909da339bee`) and the candidate with the +same wrapper rather than comparing against these numbers. Unchanged tests varied +between runs on a shared host (moq-stats' `produce::tests` by up to 40%), so credit only +the tests a change touches. + +### Done so far + +[moq-dev/moq#6914](https://github.com/moq-dev/moq/pull/6914) cut the cohort from +137.83 s to 116.83 s with fixture-only changes: the two MPEG-TS reset tests in +`container::ts::import` drain on the exporter's deadlines with a retention derived from +their stimulus instead of an hour of idle output (12.98 s to 0.18 s), and the jitter +`drift` and `feed` helpers advance a `moq_clock::Deadlines` queue instead of the tokio +paused clock (10.16 s to 6.80 s across 8 tests). About 48 s remains to cut. + +Tried and dropped, so don't redo them without new evidence: + +- Caching the jitter lead median: the frame-rate benchmark stayed within noise and one + cell regressed about 6%. +- A containment shortcut for miss-verdict intersections in moq-net: sparse gap and + range sweeps were unchanged (4096 timed gaps, 2.796 ms to 2.783 ms). The broader + algorithmic fix landed in + [moq-dev/moq#6921](https://github.com/moq-dev/moq/pull/6921). +- Retaining only one phase's span in the TS reset tests: it shed the boundary and + failed the unsignalled-reset assertion. + +### Remaining cost + +After CPU of the frozen cohort by module, largest first: + +| Module | Tests | CPU s | +| --- | ---: | ---: | +| moq-mux `container::ts::export_timing_test` | 14 | 22.2 | +| moq-net `model::track::test` | 5 | 17.3 | +| moq-mux `container::ts::recovery_test` | 18 | 11.6 | +| moq-net integration `mesh_withdraw` | 6 | 9.7 | +| moq-net integration `routing_contracts::handoff` | 3 | 8.3 | +| moq-mux `jitter::tests` | 8 | 6.8 | +| moq-quic `tests::sim` | 3 | 4.6 | +| moq-stats `produce::tests` | 4 | 4.0 | +| everything else | 42 | 32.4 | + +The moq-net `model::track::test` gap tests run through the same miss judging, which +[moq-dev/moq#6921](https://github.com/moq-dev/moq/pull/6921) made incremental after this +measurement, so re-measure them first; the export timing and recovery tests are the +largest fixture-only target. + Public API: none. diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 91d8298ea64..e115089876f 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -10144,6 +10144,41 @@ pub(super) mod test { } } + /// Each reset stimulus has eight PES packets of two 72 ms MP2 frames. + const RESET_PES: u64 = 8; + const RESET_PES_TICKS: u64 = 2 * 72 * 90; + const RESET_LEAP_TICKS: u64 = 30 * 90_000; + + /// Drain queued media on its existing deadlines, stopping when the live export is idle. + async fn exported_discontinuities( + exporter: &mut crate::container::ts::Export, + ) -> (usize, usize) { + let waiter = kio::Waiter::noop(); + let mut frames = 0; + let mut flagged = 0; + loop { + match exporter.poll_next(&waiter) { + std::task::Poll::Ready(Ok(Some(frame))) => { + frames += 1; + flagged += frame + .payload + .as_chunks::<188>() + .0 + .iter() + .filter(|p| p[3] & 0x20 != 0 && p[4] > 0 && p[5] & 0x80 != 0) + .count(); + } + std::task::Poll::Ready(Ok(None)) => break, + std::task::Poll::Ready(Err(err)) => panic!("exporter error: {err}"), + std::task::Poll::Pending => { + let Some(deadline) = exporter.next_due() else { break }; + moq_clock::sleep_until(deadline).await; + } + } + } + (frames, flagged) + } + /// Import `before`, re-export the broadcast to MPEG-TS until it has gone out, then import /// `after` and count the exported packets whose adaptation field sets /// `discontinuity_indicator`. @@ -10154,37 +10189,24 @@ pub(super) mod test { let mut import = super::Import::new(broadcast, catalog.reserve()); import.decode(&bytes::BytesMut::from(before)).unwrap(); - // `import` and `catalog` stay alive so the exporter can subscribe to the retained - // tracks. A generous delay, since the default of zero would shed the earlier groups - // and the boundary with them. The paused clock runs out the delay and every gap in - // the media without a real wait. - let delay = std::time::Duration::from_secs(3600); + // Retain the complete stimulus span, the 30 s leap included: less sheds earlier + // groups and their boundary. Queued media is drained on its deadlines, without + // generating output through an arbitrary silence before feeding the next phase. + let delay = std::time::Duration::from_millis((RESET_LEAP_TICKS + RESET_PES * RESET_PES_TICKS) / 90); let mut exporter = crate::container::ts::export_test::open_export(crate::source::announced(&consumer).await) .await .unwrap() .with_delay(delay) .with_replay(); - let flags = |frame: crate::container::Frame| { - frame - .payload - .as_chunks::<188>() - .0 - .iter() - .filter(|p| p[3] & 0x20 != 0 && p[4] > 0 && p[5] & 0x80 != 0) - .count() - }; // A discontinuity before anything went out moves no clock, so the media before the // boundary goes out first, as it would live. - let mut flagged = 0; - while let Ok(Ok(Some(frame))) = tokio::time::timeout(2 * delay, exporter.next()).await { - flagged += flags(frame); - } + let (frames, before_flags) = exported_discontinuities(&mut exporter).await; + assert!(frames > 0, "media before the boundary must reach the output"); import.decode(&bytes::BytesMut::from(after)).unwrap(); import.finish().unwrap(); - while let Ok(Ok(Some(frame))) = tokio::time::timeout(2 * delay, exporter.next()).await { - flagged += flags(frame); - } - flagged + let (frames, after_flags) = exported_discontinuities(&mut exporter).await; + assert!(frames > 0, "media after the boundary must reach the output"); + before_flags + after_flags } /// The end-to-end shape the #2833 stimulus campaign graded: source, MoQ frames, exported @@ -10196,12 +10218,10 @@ pub(super) mod test { #[tokio::test(start_paused = true)] async fn a_signalled_reset_reaches_the_exported_clock() { const PID: u16 = 0x0061; - // One PES is two 72 ms MP2 frames. - const PES_TICKS: u64 = 2 * 72 * 90; let media = |ts: &mut Vec, cc: &mut u8, base: u64| { - for i in 0..8 { - ts.extend_from_slice(&mp2_pes(PID, *cc, base + i * PES_TICKS, [0xAA, 0xBB])); + for i in 0..RESET_PES { + ts.extend_from_slice(&mp2_pes(PID, *cc, base + i * RESET_PES_TICKS, [0xAA, 0xBB])); *cc = (*cc + 1) & 0x0f; } }; @@ -10212,8 +10232,8 @@ pub(super) mod test { let mut signalled = clock_break_packet(PID); let mut unsignalled = Vec::new(); let mut peer_cc = cc; - media(&mut signalled, &mut cc, 31 * 90_000); - media(&mut unsignalled, &mut peer_cc, 31 * 90_000); + media(&mut signalled, &mut cc, 90_000 + RESET_LEAP_TICKS); + media(&mut unsignalled, &mut peer_cc, 90_000 + RESET_LEAP_TICKS); assert_eq!( export_discontinuities(&before, &signalled).await, @@ -10231,16 +10251,14 @@ pub(super) mod test { /// generation, so the clock flags it once however many tracks declared the marker. #[tokio::test(start_paused = true)] async fn a_shared_forward_boundary_resets_once_per_rendition() { - const PES_TICKS: u64 = 2 * 72 * 90; - let mut stimulus = synth_pmt( &[(StreamType::Mpeg1Audio, PCR_PID), (StreamType::Mpeg1Audio, PEER_PID)], false, ); let media = |ts: &mut Vec, cc: &mut u8, base: u64| { - for i in 0..8 { + for i in 0..RESET_PES { for pid in [PCR_PID, PEER_PID] { - ts.extend_from_slice(&mp2_pes(pid, *cc, base + i * PES_TICKS, [0xAA, 0xBB])); + ts.extend_from_slice(&mp2_pes(pid, *cc, base + i * RESET_PES_TICKS, [0xAA, 0xBB])); } *cc = (*cc + 1) & 0x0f; } @@ -10249,7 +10267,7 @@ pub(super) mod test { let mut cc = 0; media(&mut stimulus, &mut cc, 90_000); let mut after = clock_break_packet(PCR_PID); - media(&mut after, &mut cc, 31 * 90_000); + media(&mut after, &mut cc, 90_000 + RESET_LEAP_TICKS); assert_eq!( export_discontinuities(&stimulus, &after).await, diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index 55ea0a3972d..40476def9d7 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -1849,6 +1849,8 @@ mod tests { async fn drift(ppm: f64, hours: f64, settled: f64, queueing: impl Fn(f64) -> Duration) -> Run { let delay = Duration::from_millis(500); let start = moq_clock::now(); + let clock = moq_clock::Deadlines::new(start); + let _entered = clock.enter(); let mut buffer = Buffer::new(delay); let mut run = Run { err: None, @@ -1875,7 +1877,7 @@ mod tests { } let source = decode.as_secs_f64() * (1.0 + ppm * 1e-6); let arrived = start + Duration::from_secs_f64(source) + queued; - tokio::time::advance(arrived.saturating_duration_since(moq_clock::now())).await; + clock.advance(arrived.max(clock.now())); let frame = Arrival { arrived, read: arrived, @@ -2037,6 +2039,8 @@ mod tests { /// that is later, or never if `None`, in the order they reach us, until a push fails. async fn feed(shape: &Shape, ppm: f64, secs: f64, arrive: impl Fn(u16, f64) -> Option) -> Fed { let start = moq_clock::now(); + let clock = moq_clock::Deadlines::new(start); + let _entered = clock.enter(); let mut buffer = Buffer::new(shape.delay); let mut frames = Vec::new(); for &track in shape.tracks { @@ -2061,7 +2065,7 @@ mod tests { }; for (sent, track, decode) in frames { let arrived = start + Duration::from_secs_f64(sent); - tokio::time::advance(arrived.saturating_duration_since(moq_clock::now())).await; + clock.advance(arrived.max(clock.now())); let frame = Arrival { arrived, read: arrived,