diff --git a/quest/m0/nightly/receive-windows-evicted.md b/quest/m0/nightly/receive-windows-evicted.md index 23a8f17f928..69cc9ef5463 100644 --- a/quest/m0/nightly/receive-windows-evicted.md +++ b/quest/m0/nightly/receive-windows-evicted.md @@ -1,34 +1,99 @@ -# [S] The receive-windows benchmark runs to the end +# [S] The overcommitted receive-window cells run again ## Goal `moq-tokio`'s `benchmark_receive_windows` -(`rs/moq-tokio/tests/integration/receive_credit.rs:1056`) passes on `main` -instead of failing with `frame: Evicted`, so Nightly's `Rust benchmark -tests` job (`gh nightly rust-test`) passes and that night's report is -published. Fixed at the cause, never by loosening what the benchmark -measures. +(`rs/moq-tokio/tests/integration/receive_credit.rs`) measures its three +overcommitted cells again, and Nightly's `Rust benchmark tests` job +(`gh nightly rust-test`) still passes. Those cells are 256 streams, +4 MiB frames, and 4 MiB frames with a following stream window. Until +then, the benchmark skips them and prints each one with +`quest/m1/group-drops.md` as the reason. + +Every cell keeps the workload it offers. A cell whose incomplete units +fit the connection window fails on any eviction. A cell that +overcommits the window reports its delivered frames and evicted streams +alongside throughput and latency, instead of failing. ## Plan -Decided 2026-11-03 (maintainer, follow-up planning after the 2026-11-03 /quest-afk and /quest-complete runs), milestone: +Decided 2026-11-03 (maintainer): m0, as a child of +[Nightly passes on main](/quest/m0/nightly/README.md). + +Overload policy, decided 2026-11-04 (maintainer): + +- ✅ Preserve the offered workload, report delivered frames and evictions + alongside throughput and latency, and keep zero-eviction assertions only + for workloads whose incomplete units fit. +- Not chosen: keep every cell lossless by changing production sender + admission or scheduling. + +Silent loss, decided 2026-11-04 (maintainer): + +- ✅ Fix it at the source in `quest/m1/group-drops.md`, planned in + [moq-dev/moq#7890](https://github.com/moq-dev/moq/pull/7890). There, + the wire always names an abandoned group, and `track::Subscriber`'s + drop signal becomes public. +- Not chosen: a benchmark give-up timeout that declares outstanding + groups lost, or publisher-paced sending that returns in-flight credit + once a group is sent. + +Split, decided 2026-11-04 (maintainer): -- ✅ m0, as a child of [Nightly passes on main](/quest/m0/nightly/README.md) -- m1 -- m2 +- ✅ Land the cells that fit now, and skip the three overcommitted cells + until group-drops lands. This temporarily revises the original Goal's + rule against loosening what the benchmark measures, and the revision + lasts until those cells run again. +- Not chosen: keep the whole benchmark waiting on group-drops. -It failed in 3 of 3 local runs on `main`, and the job's failure skips the -rest of the night's report. Likely since -[moq-dev/moq#7597](https://github.com/moq-dev/moq/pull/7597) changed its -subscription to a track `Delay { min, max }`: confirm by running the -benchmark at #7597's parent and at #7597. +Landed in [moq-dev/moq#7838](https://github.com/moq-dev/moq/pull/7838): -- Repro: `cargo nextest run -p moq-tokio --release -E - 'test(benchmark_receive_windows)' --run-ignored only`. -- Decide whether the benchmark's subscription now asks for less than it - reads (a zero `delay.max` evicting groups the reader still wants), or - whether eviction changed under it, and fix that side. -- [Receive pool](/quest/m2/receive-pool.md) sweeps this harness later; keep - its cases runnable. +- `Cell::fits`: a cell's incomplete units fit when `streams × min(frame, + 1 MiB storage segment)` is at most the connection window. + Equal-priority tracks interleave by packet, so each can hold one frame, + or one segment of a larger frame, before any completes. The default, + 1-stream, 16 KiB-frame, and 8/32 MiB-window cells fit. They pass on + both paths with every frame delivered and nothing evicted. +- The CSV adds `delivered_frames` and `evicted_streams`. + `sh/gh/bench-report.py` reads them by exact name as measurements, not + dimensions. +- `measure` already counts an `Evicted` group as lost when a cell + doesn't fit, and it asserts zero evictions only for cells that fit. +- [moq-dev/moq#7729](https://github.com/moq-dev/moq/pull/7729)'s zero + default max age expired the benchmark's superseded in-flight groups at + the publisher. The benchmark's tracks now set no max age. + +What remains: + +- Each track paces itself on the groups its reader resolves. The + subscriber also abandons some groups without `recv_group` ever + returning them: a stream evicted before its group header was + identified, or an evicted group whose refill FETCH was evicted too + (traced as `fetch abandoned`). Such a group's in-flight permit never + returns, so the writer stalls and the reader times out. The 4 MiB-frame + cell hit this on every run, and the 256-stream cell intermittently. +- Once group-drops lands, the reader returns a dropped group's permit and + counts the group as lost. Then remove the skip (`OVERCOMMITTED` and the + `Cell::fits` partition in `benchmark_receive_windows`), and run the full + sweep through `quest gate`. +- Retain the stream/frame/window axes and the actual offered workload + for [Receive pool](/quest/m2/receive-pool.md). +- Repro: `just rs test -p moq-tokio --test integration --release -E + 'test(benchmark_receive_windows)' --run-ignored only --success-output + immediate`. + +Investigation history: the regression boundary at +[moq-dev/moq#7624](https://github.com/moq-dev/moq/pull/7624), where the +stock connection window shrank from 128 MiB to 4 MiB, and the rejected +connection-wide admission semaphore are recorded in +[moq-dev/moq#7838](https://github.com/moq-dev/moq/pull/7838). Public API: none. Wire: none. + +## Waiting + +- Outside condition: `quest/m1/group-drops.md`, planned in + [moq-dev/moq#7890](https://github.com/moq-dev/moq/pull/7890), finishes, + making `track::Subscriber`'s drop signal public on `main`. Check that + [moq-dev/moq#7890](https://github.com/moq-dev/moq/pull/7890) has + merged and that the quest file has since been deleted from `main`. diff --git a/rs/moq-tokio/tests/integration/receive_credit.rs b/rs/moq-tokio/tests/integration/receive_credit.rs index d50c03e1cd3..b411b5d996a 100644 --- a/rs/moq-tokio/tests/integration/receive_credit.rs +++ b/rs/moq-tokio/tests/integration/receive_credit.rs @@ -923,6 +923,13 @@ const BENCH_BLOAT: (u64, usize) = (200_000_000, 16 << 20); /// trip. Deeper pipelines of large frames evict: newest-first delivery starves a track's older /// groups while newer ones arrive, until their remainders fill the window. const INFLIGHT: usize = 1 << 20; +/// The receiver's storage segment (`moq_net`'s crate-private `frame::SEGMENT`): a frame within +/// one keeps its credit until it completes, and a larger one returns each segment's as it seals. +const SEGMENT: usize = 1 << 20; +/// Why cells whose incomplete units overcommit the window are skipped: the quest that makes +/// `track::Subscriber` report each group it abandons, which their readers need to return its +/// in-flight credit. +const OVERCOMMITTED: &str = "waits on quest/m1/group-drops.md"; /// One benchmark configuration. #[derive(Clone, Copy)] @@ -939,21 +946,35 @@ struct Cell { follow: bool, } +impl Cell { + /// Whether every track's incomplete unit fits the connection window at once. Equal-priority + /// tracks interleave by packet, so each can hold a whole frame, or one storage segment of a + /// larger frame, before any completes. A cell whose units fit must evict nothing; one whose + /// units overcommit the window is measured as offered, and its evictions are reported. + fn fits(&self) -> bool { + (self.streams * self.frame.min(SEGMENT)) as u64 <= self.window + } +} + /// What one benchmark run measured. struct Measured { /// The most credit the peer had spent at once: leased or not yet read. peak: u64, /// Payload bytes delivered per second. throughput: f64, - /// How long frames took from the write to the receiver completing them, sorted. + /// How long each delivered frame took from the write to the receiver completing it, sorted. latency: Vec, + /// Streams the stuck receive window evicted, each losing its group's frame. + evicted: u64, /// Datagrams the bottleneck's full queue dropped. overflowed: u64, } -/// Deliver `bytes` over `cell.streams` tracks of one-frame groups, each track keeping -/// [`INFLIGHT`] bytes outstanding. Every incomplete frame fits the window alongside the -/// others, so nothing may be evicted, through `profile` or not. +/// Offer `bytes` over `cell.streams` tracks of one-frame groups, each track keeping +/// [`INFLIGHT`] bytes outstanding, through `profile` or not. A group whose stream the stuck +/// window evicts counts as lost rather than failing the run, so an overcommitted cell is +/// measured at the load it offers; a cell whose incomplete units [fit](Cell::fits) must lose +/// nothing. async fn measure(cell: Cell, profile: Option, bytes: usize) -> Measured { let mut transport = transport(cell.window); transport.limits.max_streams = std::num::NonZeroU64::new(moq_tokio::quic::DEFAULT_MAX_STREAMS); @@ -964,7 +985,12 @@ async fn measure(cell: Cell, profile: Option, bytes: usize) let mut tracks = Vec::with_capacity(cell.streams); for index in 0..cell.streams { let name = index.to_string(); - let track = setup.broadcast.create_track(name.as_str(), timed()).unwrap(); + // A track keeps several groups in flight, so a superseded one must stay readable rather + // than expire at the publisher's default zero max age. + let track = setup + .broadcast + .create_track(name.as_str(), timed().with_max_age(None)) + .unwrap(); // Groups in flight fall behind the live edge, so a zero budget would skip them. let subscription = Subscription::default().with_delay(Delay::default().with_max(TIMEOUT)); let sub = kio::clock::timeout(TIMEOUT, setup.remote.track(&name).unwrap().subscribe(subscription)) @@ -1000,14 +1026,19 @@ async fn measure(cell: Cell, profile: Option, bytes: usize) let mut latency = Vec::with_capacity(rounds); for _ in 0..rounds { let mut group = next_group(&mut sub).await; - let frame = kio::clock::timeout(TIMEOUT, group.read_frame()) + let read = kio::clock::timeout(TIMEOUT, group.read_frame()) .await - .expect("frame timed out") - .expect("frame") - .expect("group ended"); - assert_eq!(frame.payload.len(), payload.len()); + .expect("frame timed out"); let sent = sent.lock().unwrap().remove(&group.sequence).expect("written"); - latency.push(sent.elapsed()); + match read { + Ok(Some(frame)) => { + assert_eq!(frame.payload.len(), payload.len()); + latency.push(sent.elapsed()); + } + // The stuck window evicted the group's stream, and it is not fetched again. + Err(moq_net::Error::Evicted) => {} + other => panic!("group {} ended with {other:?}", group.sequence), + } inflight.add_permits(1); } // Keep the track open until every group is read. @@ -1023,11 +1054,19 @@ async fn measure(cell: Cell, profile: Option, bytes: usize) let elapsed = start.elapsed(); latency.sort(); let credit = setup.conn.stats().credit; - assert_eq!(credit.evicted, 0, "pacing evicted a stream"); + if cell.fits() { + assert_eq!( + credit.evicted, 0, + "evicted a stream whose incomplete units fit the window" + ); + assert_eq!(latency.len(), rounds * cell.streams, "lost a frame without an eviction"); + } + assert!(!latency.is_empty(), "eviction starved every frame"); Measured { peak: credit.peak_spent, throughput: (latency.len() * cell.frame) as f64 / elapsed.as_secs_f64(), latency, + evicted: credit.evicted, overflowed: setup .verify() .map_or(0, |stats| stats.down.overflowed + stats.up.overflowed), @@ -1041,10 +1080,16 @@ async fn measure(cell: Cell, profile: Option, bytes: usize) /// size, so a cost that grows with either shows up as a slope; the connection window, which /// trades memory for throughput once it nears the path's bandwidth-delay product; and a /// stream window that follows the connection window. Reports the peak credit the peer had -/// spent (leased plus unread), throughput of delivered frames, their completion latency, the -/// bottleneck's queue drops, each the median of the runs, and fails if any run evicted a stream. The -/// publisher is a stock server, so its send window bounds what is in flight as it would in -/// production. Run by nightly. +/// spent (leased plus unread), throughput of delivered frames, their completion latency, how +/// many frames were delivered and streams evicted, and the bottleneck's queue drops, each the +/// median of the runs. Any cell whose incomplete units [fit](Cell::fits) fails if a run +/// evicted a stream. The publisher is a stock server, so its send window bounds what is in +/// flight as it would in production. Run by nightly. +/// +/// Cells whose units overcommit the window are skipped, each printed with [`OVERCOMMITTED`] +/// as the reason. They should keep their offered load and report what eviction cost them, but +/// the subscriber can abandon a group without handing it out, so its in-flight credit never +/// returns and the track's writer stalls. #[tokio::test(flavor = "multi_thread")] #[ignore = "release benchmark, run by nightly"] async fn benchmark_receive_windows() { @@ -1078,12 +1123,28 @@ async fn benchmark_receive_windows() { follow: true, ..default })); + let (cells, skipped): (Vec<_>, Vec<_>) = cells.partition(Cell::fits); + for cell in skipped { + // Not stdout: the nightly report reads every line after the table's header as a row. + eprintln!( + "skipped {} cell, {} streams of {} B frames over a {} MiB window: {OVERCOMMITTED}", + cell.axis, + cell.streams, + cell.frame, + cell.window >> 20, + ); + } let paths = [ ("loopback", None, BENCH_BYTES), ("bufferbloat", Some(bufferbloat(BENCH_BLOAT.0)), BENCH_BLOAT.1), ]; - println!("path,axis,streams,frame,window_mib,stream_window,peak_bytes,mib_per_s,p50_ms,p99_ms,overflowed_packets"); - for (cell, (path, profile, bytes)) in cells.flat_map(|cell| paths.clone().map(|path| (cell, path))) { + println!( + "path,axis,streams,frame,window_mib,stream_window,peak_bytes,mib_per_s,p50_ms,p99_ms,delivered_frames,evicted_streams,overflowed_packets" + ); + for (cell, (path, profile, bytes)) in cells + .into_iter() + .flat_map(|cell| paths.clone().map(|path| (cell, path))) + { let mut runs = Vec::with_capacity(RUNS); for _ in 0..RUNS { runs.push(measure(cell, profile.clone(), bytes).await); @@ -1101,9 +1162,11 @@ async fn benchmark_receive_windows() { }; let peak = median(runs.iter().map(|run| run.peak as f64).collect()); let throughput = median(runs.iter().map(|run| run.throughput).collect()); + let delivered = median(runs.iter().map(|run| run.latency.len() as f64).collect()); + let evicted = median(runs.iter().map(|run| run.evicted as f64).collect()); let overflowed = median(runs.iter().map(|run| run.overflowed as f64).collect()); println!( - "{path},{},{},{},{},{},{peak:.0},{:.0},{:.2},{:.2},{overflowed}", + "{path},{},{},{},{},{},{peak:.0},{:.0},{:.2},{:.2},{delivered},{evicted},{overflowed}", cell.axis, cell.streams, cell.frame, diff --git a/sh/gh/bench-report.py b/sh/gh/bench-report.py index 6e7f55c5ea6..895199677bb 100644 --- a/sh/gh/bench-report.py +++ b/sh/gh/bench-report.py @@ -15,8 +15,8 @@ from pathlib import Path from urllib.parse import quote -BIGGER = {"MB/s", "MiB/s", "B/s", "ops/s", "frames/s"} -SMALLER = {"ns", "us", "µs", "ms", "s", "ms/MB", "x", "bytes", "B", "MiB", "packets"} +BIGGER = {"MB/s", "MiB/s", "B/s", "ops/s", "frames/s", "frames"} +SMALLER = {"ns", "us", "µs", "ms", "s", "ms/MB", "x", "bytes", "B", "MiB", "packets", "streams"} def value(number): @@ -135,6 +135,11 @@ def measurement_unit(column): """The unit a benchmark CSV column measures, or None for a workload dimension.""" if column in ("size_bytes", "peak_bytes"): return "bytes" + # Named exactly: other benchmarks print frame and stream counts as workload dimensions. + if column == "delivered_frames": + return "frames" + if column == "evicted_streams": + return "streams" if column == "mib_per_s" or column.endswith("_mib_per_s"): return "MiB/s" if column.endswith("_packets"): diff --git a/sh/gh/bench-report.test.py b/sh/gh/bench-report.test.py index 363d000c51a..7fe179b659e 100644 --- a/sh/gh/bench-report.test.py +++ b/sh/gh/bench-report.test.py @@ -142,7 +142,7 @@ def test_rust_tests_record_each_benchmark_by_package_and_test(self): ( "moq-tokio::integration", "receive_credit::benchmark_receive_windows", - "path,mib_per_s,overflowed_packets\nloopback,812,0\n" + "path,mib_per_s,delivered_frames,evicted_streams,overflowed_packets\nloopback,812,1021,3,0\n" "test receive_credit::benchmark_receive_windows has been running for over 60 seconds\n", ), ( @@ -169,6 +169,16 @@ def test_rust_tests_record_each_benchmark_by_package_and_test(self): "MiB/s", 812, ), + ( + "rust-test/moq-tokio/receive_credit/benchmark_receive_windows/path=loopback/delivered_frames", + "frames", + 1021, + ), + ( + "rust-test/moq-tokio/receive_credit/benchmark_receive_windows/path=loopback/evicted_streams", + "streams", + 3, + ), ( "rust-test/moq-tokio/receive_credit/benchmark_receive_windows/path=loopback/overflowed_packets", "packets",