From 57050c3aa3022d4c6e511f13cfea60fb5cab7165 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 00:40:03 -0800 Subject: [PATCH 1/7] chore(quest): claim quest/m0/nightly/receive-windows-evicted.md Quest-Claim: 532564dcb180f176 From 032f2766fd0165b54c2b6cb5742b8dc1ea720fd5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 00:54:52 -0800 Subject: [PATCH 2/7] fix(bench): bound receive-window admission across tracks Co-Authored-By: GPT-6.1 Sol --- quest/m0/nightly/README.md | 1 - quest/m0/nightly/receive-windows-evicted.md | 34 ------------------- .../tests/integration/receive_credit.rs | 17 +++++++--- 3 files changed, 13 insertions(+), 39 deletions(-) delete mode 100644 quest/m0/nightly/receive-windows-evicted.md diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index 43a745240fb..ae1b4922805 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -35,7 +35,6 @@ Public API: none. Wire: none. ## Required -- [The receive-windows benchmark runs to the end](/quest/m0/nightly/receive-windows-evicted.md) - `benchmark_receive_windows` stops failing `frame: Evicted`, so the Rust benchmark-tests job publishes its report - [The relay's startup fits its stack under all features](/quest/m0/nightly/relay-stack.md) - `debug_relay_load_and_run_fit_windows_stack` passes under `--all-features`, fixed at the feature that grew the future - [The binding benchmark's 64 KiB case measures](/quest/m0/nightly/bindings-bench-evicted.md) - the 64 KiB case completes instead of failing `evicted` - [The range-widening bench expects a narrowed read to end](/quest/m0/nightly/track-range-widening.md) - `track_range_widening/1r/64` stops asserting a pending read that #7545 made end diff --git a/quest/m0/nightly/receive-windows-evicted.md b/quest/m0/nightly/receive-windows-evicted.md deleted file mode 100644 index 23a8f17f928..00000000000 --- a/quest/m0/nightly/receive-windows-evicted.md +++ /dev/null @@ -1,34 +0,0 @@ -# [S] The receive-windows benchmark runs to the end - -## Goal - -`moq-tokio`'s `benchmark_receive_windows` -(`rs/moq-tokio/tests/integration/receive_credit.rs:1056`) passes on `main` -instead of failing with `frame: Evicted`, so Nightly's `Rust benchmark -tests` job (`gh nightly rust-test`) passes and that night's report is -published. Fixed at the cause, never by loosening what the benchmark -measures. - -## Plan - -Decided 2026-11-03 (maintainer, follow-up planning after the 2026-11-03 /quest-afk and /quest-complete runs), milestone: - -- ✅ m0, as a child of [Nightly passes on main](/quest/m0/nightly/README.md) -- m1 -- m2 - -It failed in 3 of 3 local runs on `main`, and the job's failure skips the -rest of the night's report. Likely since -[moq-dev/moq#7597](https://github.com/moq-dev/moq/pull/7597) changed its -subscription to a track `Delay { min, max }`: confirm by running the -benchmark at #7597's parent and at #7597. - -- Repro: `cargo nextest run -p moq-tokio --release -E - 'test(benchmark_receive_windows)' --run-ignored only`. -- Decide whether the benchmark's subscription now asks for less than it - reads (a zero `delay.max` evicting groups the reader still wants), or - whether eviction changed under it, and fix that side. -- [Receive pool](/quest/m2/receive-pool.md) sweeps this harness later; keep - its cases runnable. - -Public API: none. Wire: none. diff --git a/rs/moq-tokio/tests/integration/receive_credit.rs b/rs/moq-tokio/tests/integration/receive_credit.rs index 4c1c52ae3c5..50779cd62e0 100644 --- a/rs/moq-tokio/tests/integration/receive_credit.rs +++ b/rs/moq-tokio/tests/integration/receive_credit.rs @@ -957,9 +957,9 @@ struct Measured { overflowed: u64, } -/// Deliver `bytes` over `cell.streams` tracks of one-frame groups, each track keeping -/// [`INFLIGHT`] bytes outstanding. Every incomplete frame fits the window alongside the -/// others, so nothing may be evicted, through `profile` or not. +/// Deliver `bytes` over `cell.streams` tracks of one-frame groups, with [`INFLIGHT`] +/// pacing per track. Connection-wide admission keeps their incomplete +/// storage segments and framing inside the receive window, through `profile` or not. async fn measure(cell: Cell, profile: Option, bytes: usize) -> Measured { let mut transport = transport(cell.window); transport.limits.max_streams = std::num::NonZeroU64::new(moq_tokio::quic::DEFAULT_MAX_STREAMS); @@ -982,19 +982,27 @@ async fn measure(cell: Cell, profile: Option, bytes: usize) let rounds = (bytes / cell.frame / cell.streams).max(1); let payload = Bytes::from(vec![7u8; cell.frame]); + // Credit returns at each sealed storage segment, even for a larger frame. Reserve + // the largest incomplete unit plus six maximum-width varints for its group/frame + // headers. Per-track pacing alone can offer streams * INFLIGHT against one window. + let credit = u32::try_from(cell.frame.min(moq_net::transport::MIN_RECEIVE_WINDOW as usize) + 6 * 8).unwrap(); + let connection = Arc::new(tokio::sync::Semaphore::new(cell.window.try_into().unwrap())); let start = kio::clock::now(); let tasks: Vec<_> = tracks .into_iter() .map(|(track, mut sub)| { let payload = payload.clone(); + let connection = connection.clone(); tokio::spawn(async move { let inflight = Arc::new(tokio::sync::Semaphore::new((INFLIGHT / payload.len()).max(1))); let sent = Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())); let writer = tokio::spawn({ - let (inflight, sent, payload) = (inflight.clone(), sent.clone(), payload.clone()); + let (inflight, sent, payload, connection) = + (inflight.clone(), sent.clone(), payload.clone(), connection.clone()); async move { for _ in 0..rounds { inflight.acquire().await.unwrap().forget(); + connection.acquire_many(credit).await.unwrap().forget(); let mut group = track.append_group().unwrap(); sent.lock().unwrap().insert(group.sequence, kio::clock::now()); group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); @@ -1015,6 +1023,7 @@ async fn measure(cell: Cell, profile: Option, bytes: usize) let sent = sent.lock().unwrap().remove(&group.sequence).expect("written"); latency.push(sent.elapsed()); inflight.add_permits(1); + connection.add_permits(credit as usize); } // Keep the track open until every group is read. drop(writer.await.unwrap()); From 60dde6119675fd91691f6f97986364934a66bd0d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 01:08:58 -0800 Subject: [PATCH 3/7] docs(quest): hold receive-window repair for overload policy Revert application admission after local review because it weakens the measured pressure. Preserve the quest with the reproduced regression boundary and the maintainer decision needed before implementation. Co-Authored-By: GPT-6.1 Sol --- quest/m0/nightly/README.md | 1 + quest/m0/nightly/receive-windows-evicted.md | 53 +++++++++++++++++++ .../tests/integration/receive_credit.rs | 17 ++---- 3 files changed, 58 insertions(+), 13 deletions(-) create mode 100644 quest/m0/nightly/receive-windows-evicted.md diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index ae1b4922805..43a745240fb 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -35,6 +35,7 @@ Public API: none. Wire: none. ## Required +- [The receive-windows benchmark runs to the end](/quest/m0/nightly/receive-windows-evicted.md) - `benchmark_receive_windows` stops failing `frame: Evicted`, so the Rust benchmark-tests job publishes its report - [The relay's startup fits its stack under all features](/quest/m0/nightly/relay-stack.md) - `debug_relay_load_and_run_fit_windows_stack` passes under `--all-features`, fixed at the feature that grew the future - [The binding benchmark's 64 KiB case measures](/quest/m0/nightly/bindings-bench-evicted.md) - the 64 KiB case completes instead of failing `evicted` - [The range-widening bench expects a narrowed read to end](/quest/m0/nightly/track-range-widening.md) - `track_range_widening/1r/64` stops asserting a pending read that #7545 made end diff --git a/quest/m0/nightly/receive-windows-evicted.md b/quest/m0/nightly/receive-windows-evicted.md new file mode 100644 index 00000000000..772c4af398c --- /dev/null +++ b/quest/m0/nightly/receive-windows-evicted.md @@ -0,0 +1,53 @@ +# [S] The receive-windows benchmark runs to the end + +## Goal + +`moq-tokio`'s `benchmark_receive_windows` +(`rs/moq-tokio/tests/integration/receive_credit.rs:1056`) passes on `main` +instead of failing with `frame: Evicted`, so Nightly's `Rust benchmark +tests` job (`gh nightly rust-test`) passes and that night's report is +published. Fixed at the cause, never by loosening what the benchmark +measures. + +## Plan + +Decided 2026-11-03 (maintainer): m0, as a child of +[Nightly passes on main](/quest/m0/nightly/README.md). + +Investigated 2026-11-04 in +[moq-dev/moq#7838](https://github.com/moq-dev/moq/pull/7838): + +- Unchanged main reproduces `frame: Evicted`. Both the merge of + [moq-dev/moq#7597](https://github.com/moq-dev/moq/pull/7597) and its + parent fail too. The subscription already asks for a 10-second + `delay.max`; the Delay API migration did not introduce this failure. +- [moq-dev/moq#7624](https://github.com/moq-dev/moq/pull/7624) reduced + the stock connection receive window from 128 MiB to 4 MiB. The harness + still permits 1 MiB per track, or one larger frame, across up to 256 + tracks. The sender interleaves equal-priority tracks by packet, so + incomplete storage segments can exhaust the connection window before + any can seal. Eviction then frees the stuck credit by design. + The unchanged sweep passes all 16 rows at its + [parent](https://github.com/moq-dev/moq/commit/097dfc42257fc893dee5116006a265660fd71b01), + then fails `frame: Evicted` in the 4 MiB frame cell at its + [merge](https://github.com/moq-dev/moq/commit/cc3c1961d849dd536dc203724bf6b6cca46f0d88). +- A connection-wide semaphore makes all 16 rows pass, three runs each, + and Nightly's report parser produces 80 measurements. It also admits + only 15 tracks' 256 KiB frames, or three tracks' 4 MiB frames, at once + with the stock window. Local review rejected this change because it + alters the measured pressure and excludes admission wait from frame + latency. The implementation was removed; no benchmark or production + code changes remain in this PR. +- Retain the stream/frame/window axes and actual offered workload for + [Receive pool](/quest/m2/receive-pool.md). Resolve the policy below + before changing either the benchmark's acceptance or production + scheduling. +- Repro: `just rs test -p moq-tokio --test integration --release -E + 'test(benchmark_receive_windows)' --run-ignored only --success-output + immediate`. Run expensive builds and sweeps through `quest gate`. + +Public API: none. Wire: none. + +## Waiting + +- Maintainer: decide whether deliberately overcommitted cells should measure loss or remain lossless. This changes benchmark acceptance or production scheduling policy, which the implementation agent cannot select while the maintainer is AFK. (1) Recommended: preserve the offered workload, report delivered frames and evictions alongside throughput/latency, and retain zero-eviction assertions for workloads whose incomplete units fit. This requires revising the Goal's acceptance explicitly. (2) Keep every existing cell lossless at the current windows and change production sender admission/scheduling so an incomplete unit can finish before the window fills; measure the effect on equal-priority fairness and latency. Check this wait by recording the maintainer's pick in this Plan and the PR; automated reproductions and the rejected semaphore cannot decide that policy. diff --git a/rs/moq-tokio/tests/integration/receive_credit.rs b/rs/moq-tokio/tests/integration/receive_credit.rs index 50779cd62e0..4c1c52ae3c5 100644 --- a/rs/moq-tokio/tests/integration/receive_credit.rs +++ b/rs/moq-tokio/tests/integration/receive_credit.rs @@ -957,9 +957,9 @@ struct Measured { overflowed: u64, } -/// Deliver `bytes` over `cell.streams` tracks of one-frame groups, with [`INFLIGHT`] -/// pacing per track. Connection-wide admission keeps their incomplete -/// storage segments and framing inside the receive window, through `profile` or not. +/// Deliver `bytes` over `cell.streams` tracks of one-frame groups, each track keeping +/// [`INFLIGHT`] bytes outstanding. Every incomplete frame fits the window alongside the +/// others, so nothing may be evicted, through `profile` or not. async fn measure(cell: Cell, profile: Option, bytes: usize) -> Measured { let mut transport = transport(cell.window); transport.limits.max_streams = std::num::NonZeroU64::new(moq_tokio::quic::DEFAULT_MAX_STREAMS); @@ -982,27 +982,19 @@ async fn measure(cell: Cell, profile: Option, bytes: usize) let rounds = (bytes / cell.frame / cell.streams).max(1); let payload = Bytes::from(vec![7u8; cell.frame]); - // Credit returns at each sealed storage segment, even for a larger frame. Reserve - // the largest incomplete unit plus six maximum-width varints for its group/frame - // headers. Per-track pacing alone can offer streams * INFLIGHT against one window. - let credit = u32::try_from(cell.frame.min(moq_net::transport::MIN_RECEIVE_WINDOW as usize) + 6 * 8).unwrap(); - let connection = Arc::new(tokio::sync::Semaphore::new(cell.window.try_into().unwrap())); let start = kio::clock::now(); let tasks: Vec<_> = tracks .into_iter() .map(|(track, mut sub)| { let payload = payload.clone(); - let connection = connection.clone(); tokio::spawn(async move { let inflight = Arc::new(tokio::sync::Semaphore::new((INFLIGHT / payload.len()).max(1))); let sent = Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())); let writer = tokio::spawn({ - let (inflight, sent, payload, connection) = - (inflight.clone(), sent.clone(), payload.clone(), connection.clone()); + let (inflight, sent, payload) = (inflight.clone(), sent.clone(), payload.clone()); async move { for _ in 0..rounds { inflight.acquire().await.unwrap().forget(); - connection.acquire_many(credit).await.unwrap().forget(); let mut group = track.append_group().unwrap(); sent.lock().unwrap().insert(group.sequence, kio::clock::now()); group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); @@ -1023,7 +1015,6 @@ async fn measure(cell: Cell, profile: Option, bytes: usize) let sent = sent.lock().unwrap().remove(&group.sequence).expect("written"); latency.push(sent.elapsed()); inflight.add_permits(1); - connection.add_permits(credit as usize); } // Keep the track open until every group is read. drop(writer.await.unwrap()); From 647f87741dc0231a3bda1f123169a84fd3e9c833 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 01:12:49 -0800 Subject: [PATCH 4/7] docs(quest): keep the admission constraint durable Apply the local reviewer suggestion: record why admission is rejected without referring to the current PR diff. Co-Authored-By: GPT-6.1 Sol --- quest/m0/nightly/receive-windows-evicted.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/quest/m0/nightly/receive-windows-evicted.md b/quest/m0/nightly/receive-windows-evicted.md index 772c4af398c..9c02f5360da 100644 --- a/quest/m0/nightly/receive-windows-evicted.md +++ b/quest/m0/nightly/receive-windows-evicted.md @@ -34,10 +34,9 @@ Investigated 2026-11-04 in - A connection-wide semaphore makes all 16 rows pass, three runs each, and Nightly's report parser produces 80 measurements. It also admits only 15 tracks' 256 KiB frames, or three tracks' 4 MiB frames, at once - with the stock window. Local review rejected this change because it - alters the measured pressure and excludes admission wait from frame - latency. The implementation was removed; no benchmark or production - code changes remain in this PR. + with the stock window. A connection-wide admission semaphore is + rejected: it changes the measured pressure and leaves admission wait + out of frame latency. - Retain the stream/frame/window axes and actual offered workload for [Receive pool](/quest/m2/receive-pool.md). Resolve the policy below before changing either the benchmark's acceptance or production From d7031a3f2c86837fb8139aae9f1f807a0266c041 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 08:39:03 -0800 Subject: [PATCH 5/7] fix(bench): report receive-window evictions instead of failing overcommitted cells Record the maintainer's 2026-11-04 overload policy: keep the offered workload, report delivered frames and evicted streams, and assert zero evictions only for cells whose incomplete units fit the connection window. The benchmark's tracks also set no max age again: #7729's zero default expired superseded in-flight groups at the publisher, so even the default cell timed out. Overcommitted cells still stall on groups the subscriber gives up on without handing them out; the quest records that open decision. Co-Authored-By: Claude Opus 5.5 --- quest/m0/nightly/receive-windows-evicted.md | 57 ++++++++++++-- .../tests/integration/receive_credit.rs | 75 ++++++++++++++----- sh/gh/bench-report.py | 9 ++- sh/gh/bench-report.test.py | 12 ++- 4 files changed, 128 insertions(+), 25 deletions(-) diff --git a/quest/m0/nightly/receive-windows-evicted.md b/quest/m0/nightly/receive-windows-evicted.md index 9c02f5360da..4a5b8f7a77b 100644 --- a/quest/m0/nightly/receive-windows-evicted.md +++ b/quest/m0/nightly/receive-windows-evicted.md @@ -3,11 +3,13 @@ ## Goal `moq-tokio`'s `benchmark_receive_windows` -(`rs/moq-tokio/tests/integration/receive_credit.rs:1056`) passes on `main` +(`rs/moq-tokio/tests/integration/receive_credit.rs`) passes on `main` instead of failing with `frame: Evicted`, so Nightly's `Rust benchmark tests` job (`gh nightly rust-test`) passes and that night's report is -published. Fixed at the cause, never by loosening what the benchmark -measures. +published. Every cell keeps the workload it offers. A cell whose +incomplete units fit the connection window still fails on any eviction. +A cell that overcommits the window reports its delivered frames and +evicted streams alongside throughput and latency, instead of failing. ## Plan @@ -45,8 +47,53 @@ Investigated 2026-11-04 in 'test(benchmark_receive_windows)' --run-ignored only --success-output immediate`. Run expensive builds and sweeps through `quest gate`. -Public API: none. Wire: none. +Overload policy, decided 2026-11-04 (maintainer): + +- ✅ Preserve the offered workload, report delivered frames and evictions + alongside throughput and latency, and keep zero-eviction assertions only + for workloads whose incomplete units fit. +- Not chosen: keep every cell lossless by changing production sender + admission or scheduling. + +Implemented in [moq-dev/moq#7838](https://github.com/moq-dev/moq/pull/7838): + +- A cell's incomplete units fit when `streams × min(frame, 1 MiB storage + segment)` is at most the connection window. That holds for the default, + 1-stream, 16 KiB-frame, and 8/32 MiB-window cells. All five pass on + both paths with every frame delivered and nothing evicted. The 256-stream, + 4 MiB-frame, and follow cells overcommit. +- The CSV adds `delivered_frames` and `evicted_streams`. + `sh/gh/bench-report.py` reads them as measurements, not as dimensions + in the series name. +- [moq-dev/moq#7729](https://github.com/moq-dev/moq/pull/7729) made + authored tracks default to a zero max age. The benchmark's tracks keep + up to 1 MiB of groups in flight, so their superseded groups expired at + the publisher and never arrived. Even the default cell timed out. The + benchmark's tracks now set no max age, as before #7729. + +Overcommitted cells still hang. A track paces itself on groups its reader +resolves, either delivered or `Evicted`. The subscriber also gives up on +some groups without `recv_group` ever returning them: a stream evicted +before its group header was identified, or an evicted group whose refill +FETCH was evicted too ("fetch abandoned"). No public API reports such a +group, so its in-flight permit never returns, the track's writer stalls, +and the reader times out. The 4 MiB-frame cell hits this on every run; +the 256-stream cell intermittently. + +Public API: none so far. Wire: none. ## Waiting -- Maintainer: decide whether deliberately overcommitted cells should measure loss or remain lossless. This changes benchmark acceptance or production scheduling policy, which the implementation agent cannot select while the maintainer is AFK. (1) Recommended: preserve the offered workload, report delivered frames and evictions alongside throughput/latency, and retain zero-eviction assertions for workloads whose incomplete units fit. This requires revising the Goal's acceptance explicitly. (2) Keep every existing cell lossless at the current windows and change production sender admission/scheduling so an incomplete unit can finish before the window fills; measure the effect on equal-priority fairness and latency. Check this wait by recording the maintainer's pick in this Plan and the PR; automated reproductions and the rejected semaphore cannot decide that policy. +- Maintainer: decide how the benchmark accounts for a group the + subscriber silently gives up on, so overcommitted cells finish at + their offered load. Options are in the PR's `## Decisions needed`. + (1) Recommended: moq-net surfaces the loss. `recv_group` hands out a + given-up group whose reads fail `Evicted`, or the subscriber counts + given-up groups. This is a public behavior change. Production readers + then learn of a loss as they already do for an identified evicted + group. (2) Benchmark-only: declare a track's outstanding group lost + after the subscriber's give-up interval. This is time-based, and the + wait inflates elapsed time, deflating throughput. (3) Benchmark-only: release + in-flight credit once the publisher finishes sending a group, not once + the reader resolves it. This changes the offered-load model. Check + this wait by recording the pick here and in the PR. diff --git a/rs/moq-tokio/tests/integration/receive_credit.rs b/rs/moq-tokio/tests/integration/receive_credit.rs index d50c03e1cd3..02c259782bc 100644 --- a/rs/moq-tokio/tests/integration/receive_credit.rs +++ b/rs/moq-tokio/tests/integration/receive_credit.rs @@ -923,6 +923,9 @@ const BENCH_BLOAT: (u64, usize) = (200_000_000, 16 << 20); /// trip. Deeper pipelines of large frames evict: newest-first delivery starves a track's older /// groups while newer ones arrive, until their remainders fill the window. const INFLIGHT: usize = 1 << 20; +/// The receiver's storage segment (`moq_net`'s crate-private `frame::SEGMENT`): a frame within +/// one keeps its credit until it completes, and a larger one returns each segment's as it seals. +const SEGMENT: usize = 1 << 20; /// One benchmark configuration. #[derive(Clone, Copy)] @@ -939,21 +942,35 @@ struct Cell { follow: bool, } +impl Cell { + /// Whether every track's incomplete unit fits the connection window at once. Equal-priority + /// tracks interleave by packet, so each can hold a whole frame, or one storage segment of a + /// larger frame, before any completes. A cell whose units fit must evict nothing; one whose + /// units overcommit the window is measured as offered, and its evictions are reported. + fn fits(&self) -> bool { + (self.streams * self.frame.min(SEGMENT)) as u64 <= self.window + } +} + /// What one benchmark run measured. struct Measured { /// The most credit the peer had spent at once: leased or not yet read. peak: u64, /// Payload bytes delivered per second. throughput: f64, - /// How long frames took from the write to the receiver completing them, sorted. + /// How long each delivered frame took from the write to the receiver completing it, sorted. latency: Vec, + /// Streams the stuck receive window evicted, each losing its group's frame. + evicted: u64, /// Datagrams the bottleneck's full queue dropped. overflowed: u64, } -/// Deliver `bytes` over `cell.streams` tracks of one-frame groups, each track keeping -/// [`INFLIGHT`] bytes outstanding. Every incomplete frame fits the window alongside the -/// others, so nothing may be evicted, through `profile` or not. +/// Offer `bytes` over `cell.streams` tracks of one-frame groups, each track keeping +/// [`INFLIGHT`] bytes outstanding, through `profile` or not. A group whose stream the stuck +/// window evicts counts as lost rather than failing the run, so an overcommitted cell is +/// measured at the load it offers; a cell whose incomplete units [fit](Cell::fits) must lose +/// nothing. async fn measure(cell: Cell, profile: Option, bytes: usize) -> Measured { let mut transport = transport(cell.window); transport.limits.max_streams = std::num::NonZeroU64::new(moq_tokio::quic::DEFAULT_MAX_STREAMS); @@ -964,7 +981,12 @@ async fn measure(cell: Cell, profile: Option, bytes: usize) let mut tracks = Vec::with_capacity(cell.streams); for index in 0..cell.streams { let name = index.to_string(); - let track = setup.broadcast.create_track(name.as_str(), timed()).unwrap(); + // A track keeps several groups in flight, so a superseded one must stay readable rather + // than expire at the publisher's default zero max age. + let track = setup + .broadcast + .create_track(name.as_str(), timed().with_max_age(None)) + .unwrap(); // Groups in flight fall behind the live edge, so a zero budget would skip them. let subscription = Subscription::default().with_delay(Delay::default().with_max(TIMEOUT)); let sub = kio::clock::timeout(TIMEOUT, setup.remote.track(&name).unwrap().subscribe(subscription)) @@ -1000,14 +1022,19 @@ async fn measure(cell: Cell, profile: Option, bytes: usize) let mut latency = Vec::with_capacity(rounds); for _ in 0..rounds { let mut group = next_group(&mut sub).await; - let frame = kio::clock::timeout(TIMEOUT, group.read_frame()) + let read = kio::clock::timeout(TIMEOUT, group.read_frame()) .await - .expect("frame timed out") - .expect("frame") - .expect("group ended"); - assert_eq!(frame.payload.len(), payload.len()); + .expect("frame timed out"); let sent = sent.lock().unwrap().remove(&group.sequence).expect("written"); - latency.push(sent.elapsed()); + match read { + Ok(Some(frame)) => { + assert_eq!(frame.payload.len(), payload.len()); + latency.push(sent.elapsed()); + } + // The stuck window evicted the group's stream, and it is not fetched again. + Err(moq_net::Error::Evicted) => {} + other => panic!("group {} ended with {other:?}", group.sequence), + } inflight.add_permits(1); } // Keep the track open until every group is read. @@ -1023,11 +1050,19 @@ async fn measure(cell: Cell, profile: Option, bytes: usize) let elapsed = start.elapsed(); latency.sort(); let credit = setup.conn.stats().credit; - assert_eq!(credit.evicted, 0, "pacing evicted a stream"); + if cell.fits() { + assert_eq!( + credit.evicted, 0, + "evicted a stream whose incomplete units fit the window" + ); + assert_eq!(latency.len(), rounds * cell.streams, "lost a frame without an eviction"); + } + assert!(!latency.is_empty(), "eviction starved every frame"); Measured { peak: credit.peak_spent, throughput: (latency.len() * cell.frame) as f64 / elapsed.as_secs_f64(), latency, + evicted: credit.evicted, overflowed: setup .verify() .map_or(0, |stats| stats.down.overflowed + stats.up.overflowed), @@ -1041,9 +1076,11 @@ async fn measure(cell: Cell, profile: Option, bytes: usize) /// size, so a cost that grows with either shows up as a slope; the connection window, which /// trades memory for throughput once it nears the path's bandwidth-delay product; and a /// stream window that follows the connection window. Reports the peak credit the peer had -/// spent (leased plus unread), throughput of delivered frames, their completion latency, the -/// bottleneck's queue drops, each the median of the runs, and fails if any run evicted a stream. The -/// publisher is a stock server, so its send window bounds what is in flight as it would in +/// spent (leased plus unread), throughput of delivered frames, their completion latency, how +/// many frames were delivered and streams evicted, and the bottleneck's queue drops, each the +/// median of the runs. Cells whose incomplete units overcommit the window keep their offered +/// load and report what eviction cost them; any other cell fails if a run evicted a stream. +/// The publisher is a stock server, so its send window bounds what is in flight as it would in /// production. Run by nightly. #[tokio::test(flavor = "multi_thread")] #[ignore = "release benchmark, run by nightly"] @@ -1082,7 +1119,9 @@ async fn benchmark_receive_windows() { ("loopback", None, BENCH_BYTES), ("bufferbloat", Some(bufferbloat(BENCH_BLOAT.0)), BENCH_BLOAT.1), ]; - println!("path,axis,streams,frame,window_mib,stream_window,peak_bytes,mib_per_s,p50_ms,p99_ms,overflowed_packets"); + println!( + "path,axis,streams,frame,window_mib,stream_window,peak_bytes,mib_per_s,p50_ms,p99_ms,delivered_frames,evicted_streams,overflowed_packets" + ); for (cell, (path, profile, bytes)) in cells.flat_map(|cell| paths.clone().map(|path| (cell, path))) { let mut runs = Vec::with_capacity(RUNS); for _ in 0..RUNS { @@ -1101,9 +1140,11 @@ async fn benchmark_receive_windows() { }; let peak = median(runs.iter().map(|run| run.peak as f64).collect()); let throughput = median(runs.iter().map(|run| run.throughput).collect()); + let delivered = median(runs.iter().map(|run| run.latency.len() as f64).collect()); + let evicted = median(runs.iter().map(|run| run.evicted as f64).collect()); let overflowed = median(runs.iter().map(|run| run.overflowed as f64).collect()); println!( - "{path},{},{},{},{},{},{peak:.0},{:.0},{:.2},{:.2},{overflowed}", + "{path},{},{},{},{},{},{peak:.0},{:.0},{:.2},{:.2},{delivered},{evicted},{overflowed}", cell.axis, cell.streams, cell.frame, diff --git a/sh/gh/bench-report.py b/sh/gh/bench-report.py index 6e7f55c5ea6..895199677bb 100644 --- a/sh/gh/bench-report.py +++ b/sh/gh/bench-report.py @@ -15,8 +15,8 @@ from pathlib import Path from urllib.parse import quote -BIGGER = {"MB/s", "MiB/s", "B/s", "ops/s", "frames/s"} -SMALLER = {"ns", "us", "µs", "ms", "s", "ms/MB", "x", "bytes", "B", "MiB", "packets"} +BIGGER = {"MB/s", "MiB/s", "B/s", "ops/s", "frames/s", "frames"} +SMALLER = {"ns", "us", "µs", "ms", "s", "ms/MB", "x", "bytes", "B", "MiB", "packets", "streams"} def value(number): @@ -135,6 +135,11 @@ def measurement_unit(column): """The unit a benchmark CSV column measures, or None for a workload dimension.""" if column in ("size_bytes", "peak_bytes"): return "bytes" + # Named exactly: other benchmarks print frame and stream counts as workload dimensions. + if column == "delivered_frames": + return "frames" + if column == "evicted_streams": + return "streams" if column == "mib_per_s" or column.endswith("_mib_per_s"): return "MiB/s" if column.endswith("_packets"): diff --git a/sh/gh/bench-report.test.py b/sh/gh/bench-report.test.py index 363d000c51a..7fe179b659e 100644 --- a/sh/gh/bench-report.test.py +++ b/sh/gh/bench-report.test.py @@ -142,7 +142,7 @@ def test_rust_tests_record_each_benchmark_by_package_and_test(self): ( "moq-tokio::integration", "receive_credit::benchmark_receive_windows", - "path,mib_per_s,overflowed_packets\nloopback,812,0\n" + "path,mib_per_s,delivered_frames,evicted_streams,overflowed_packets\nloopback,812,1021,3,0\n" "test receive_credit::benchmark_receive_windows has been running for over 60 seconds\n", ), ( @@ -169,6 +169,16 @@ def test_rust_tests_record_each_benchmark_by_package_and_test(self): "MiB/s", 812, ), + ( + "rust-test/moq-tokio/receive_credit/benchmark_receive_windows/path=loopback/delivered_frames", + "frames", + 1021, + ), + ( + "rust-test/moq-tokio/receive_credit/benchmark_receive_windows/path=loopback/evicted_streams", + "streams", + 3, + ), ( "rust-test/moq-tokio/receive_credit/benchmark_receive_windows/path=loopback/overflowed_packets", "packets", From b776784275fe0583b7e41c7e8244bd238d320641 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 08:52:22 -0800 Subject: [PATCH 6/7] docs(quest): overcommitted receive-window cells wait on group drops The maintainer chose, 2026-11-04, to report abandoned groups at the source (quest/m1/group-drops.md) over a benchmark give-up timeout or publisher-paced sending. Co-Authored-By: Claude Opus 5.5 --- quest/m0/nightly/receive-windows-evicted.md | 32 ++++++++++++--------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/quest/m0/nightly/receive-windows-evicted.md b/quest/m0/nightly/receive-windows-evicted.md index 4a5b8f7a77b..91edab7f2bf 100644 --- a/quest/m0/nightly/receive-windows-evicted.md +++ b/quest/m0/nightly/receive-windows-evicted.md @@ -80,20 +80,24 @@ group, so its in-flight permit never returns, the track's writer stalls, and the reader times out. The 4 MiB-frame cell hits this on every run; the 256-stream cell intermittently. -Public API: none so far. Wire: none. +Silent loss, decided 2026-11-04 (maintainer): + +- ✅ Fix it at the source. Overcommitted cells wait on the m1 quest + `quest/m1/group-drops.md`, planned in + [moq-dev/moq#7890](https://github.com/moq-dev/moq/pull/7890). There, the wire always names an abandoned group, and + `track::Subscriber`'s drop signal becomes public. Once it lands, each + track's reader returns a dropped group's in-flight permit and counts the + group as lost, so overcommitted cells finish at the load they offer. +- Not chosen: a benchmark give-up timeout that declares outstanding + groups lost, or publisher-paced sending that returns in-flight credit + once a group is sent. + +Public API: none. Wire: none. ## Waiting -- Maintainer: decide how the benchmark accounts for a group the - subscriber silently gives up on, so overcommitted cells finish at - their offered load. Options are in the PR's `## Decisions needed`. - (1) Recommended: moq-net surfaces the loss. `recv_group` hands out a - given-up group whose reads fail `Evicted`, or the subscriber counts - given-up groups. This is a public behavior change. Production readers - then learn of a loss as they already do for an identified evicted - group. (2) Benchmark-only: declare a track's outstanding group lost - after the subscriber's give-up interval. This is time-based, and the - wait inflates elapsed time, deflating throughput. (3) Benchmark-only: release - in-flight credit once the publisher finishes sending a group, not once - the reader resolves it. This changes the offered-load model. Check - this wait by recording the pick here and in the PR. +- Outside condition: the m1 quest `quest/m1/group-drops.md` (planned in + [moq-dev/moq#7890](https://github.com/moq-dev/moq/pull/7890)) finishes, + so `track::Subscriber`'s drop signal is public on `main`. Check it this + way: #7890 has merged, and that quest file has since been deleted from + `main`. From dec38b16baa4f371378c0700638058c3576f895c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 09:04:21 -0800 Subject: [PATCH 7/7] fix(bench): skip overcommitted receive-window cells until group drops land The maintainer chose, 2026-11-04, to land the cells whose incomplete units fit and to skip the 256-stream, 4 MiB-frame, and follow cells until quest/m1/group-drops.md makes abandoned groups visible to readers. Each skipped cell prints with that reason on stderr, so the nightly report parser keeps reading stdout as the CSV table. The quest now tracks re-enabling those cells. Co-Authored-By: Claude Opus 5.5 --- quest/m0/nightly/receive-windows-evicted.md | 146 +++++++++--------- .../tests/integration/receive_credit.rs | 32 +++- 2 files changed, 98 insertions(+), 80 deletions(-) diff --git a/quest/m0/nightly/receive-windows-evicted.md b/quest/m0/nightly/receive-windows-evicted.md index 91edab7f2bf..69cc9ef5463 100644 --- a/quest/m0/nightly/receive-windows-evicted.md +++ b/quest/m0/nightly/receive-windows-evicted.md @@ -1,52 +1,25 @@ -# [S] The receive-windows benchmark runs to the end +# [S] The overcommitted receive-window cells run again ## Goal `moq-tokio`'s `benchmark_receive_windows` -(`rs/moq-tokio/tests/integration/receive_credit.rs`) passes on `main` -instead of failing with `frame: Evicted`, so Nightly's `Rust benchmark -tests` job (`gh nightly rust-test`) passes and that night's report is -published. Every cell keeps the workload it offers. A cell whose -incomplete units fit the connection window still fails on any eviction. -A cell that overcommits the window reports its delivered frames and -evicted streams alongside throughput and latency, instead of failing. +(`rs/moq-tokio/tests/integration/receive_credit.rs`) measures its three +overcommitted cells again, and Nightly's `Rust benchmark tests` job +(`gh nightly rust-test`) still passes. Those cells are 256 streams, +4 MiB frames, and 4 MiB frames with a following stream window. Until +then, the benchmark skips them and prints each one with +`quest/m1/group-drops.md` as the reason. + +Every cell keeps the workload it offers. A cell whose incomplete units +fit the connection window fails on any eviction. A cell that +overcommits the window reports its delivered frames and evicted streams +alongside throughput and latency, instead of failing. ## Plan Decided 2026-11-03 (maintainer): m0, as a child of [Nightly passes on main](/quest/m0/nightly/README.md). -Investigated 2026-11-04 in -[moq-dev/moq#7838](https://github.com/moq-dev/moq/pull/7838): - -- Unchanged main reproduces `frame: Evicted`. Both the merge of - [moq-dev/moq#7597](https://github.com/moq-dev/moq/pull/7597) and its - parent fail too. The subscription already asks for a 10-second - `delay.max`; the Delay API migration did not introduce this failure. -- [moq-dev/moq#7624](https://github.com/moq-dev/moq/pull/7624) reduced - the stock connection receive window from 128 MiB to 4 MiB. The harness - still permits 1 MiB per track, or one larger frame, across up to 256 - tracks. The sender interleaves equal-priority tracks by packet, so - incomplete storage segments can exhaust the connection window before - any can seal. Eviction then frees the stuck credit by design. - The unchanged sweep passes all 16 rows at its - [parent](https://github.com/moq-dev/moq/commit/097dfc42257fc893dee5116006a265660fd71b01), - then fails `frame: Evicted` in the 4 MiB frame cell at its - [merge](https://github.com/moq-dev/moq/commit/cc3c1961d849dd536dc203724bf6b6cca46f0d88). -- A connection-wide semaphore makes all 16 rows pass, three runs each, - and Nightly's report parser produces 80 measurements. It also admits - only 15 tracks' 256 KiB frames, or three tracks' 4 MiB frames, at once - with the stock window. A connection-wide admission semaphore is - rejected: it changes the measured pressure and leaves admission wait - out of frame latency. -- Retain the stream/frame/window axes and actual offered workload for - [Receive pool](/quest/m2/receive-pool.md). Resolve the policy below - before changing either the benchmark's acceptance or production - scheduling. -- Repro: `just rs test -p moq-tokio --test integration --release -E - 'test(benchmark_receive_windows)' --run-ignored only --success-output - immediate`. Run expensive builds and sweeps through `quest gate`. - Overload policy, decided 2026-11-04 (maintainer): - ✅ Preserve the offered workload, report delivered frames and evictions @@ -55,49 +28,72 @@ Overload policy, decided 2026-11-04 (maintainer): - Not chosen: keep every cell lossless by changing production sender admission or scheduling. -Implemented in [moq-dev/moq#7838](https://github.com/moq-dev/moq/pull/7838): - -- A cell's incomplete units fit when `streams × min(frame, 1 MiB storage - segment)` is at most the connection window. That holds for the default, - 1-stream, 16 KiB-frame, and 8/32 MiB-window cells. All five pass on - both paths with every frame delivered and nothing evicted. The 256-stream, - 4 MiB-frame, and follow cells overcommit. -- The CSV adds `delivered_frames` and `evicted_streams`. - `sh/gh/bench-report.py` reads them as measurements, not as dimensions - in the series name. -- [moq-dev/moq#7729](https://github.com/moq-dev/moq/pull/7729) made - authored tracks default to a zero max age. The benchmark's tracks keep - up to 1 MiB of groups in flight, so their superseded groups expired at - the publisher and never arrived. Even the default cell timed out. The - benchmark's tracks now set no max age, as before #7729. - -Overcommitted cells still hang. A track paces itself on groups its reader -resolves, either delivered or `Evicted`. The subscriber also gives up on -some groups without `recv_group` ever returning them: a stream evicted -before its group header was identified, or an evicted group whose refill -FETCH was evicted too ("fetch abandoned"). No public API reports such a -group, so its in-flight permit never returns, the track's writer stalls, -and the reader times out. The 4 MiB-frame cell hits this on every run; -the 256-stream cell intermittently. - Silent loss, decided 2026-11-04 (maintainer): -- ✅ Fix it at the source. Overcommitted cells wait on the m1 quest - `quest/m1/group-drops.md`, planned in - [moq-dev/moq#7890](https://github.com/moq-dev/moq/pull/7890). There, the wire always names an abandoned group, and - `track::Subscriber`'s drop signal becomes public. Once it lands, each - track's reader returns a dropped group's in-flight permit and counts the - group as lost, so overcommitted cells finish at the load they offer. +- ✅ Fix it at the source in `quest/m1/group-drops.md`, planned in + [moq-dev/moq#7890](https://github.com/moq-dev/moq/pull/7890). There, + the wire always names an abandoned group, and `track::Subscriber`'s + drop signal becomes public. - Not chosen: a benchmark give-up timeout that declares outstanding groups lost, or publisher-paced sending that returns in-flight credit once a group is sent. +Split, decided 2026-11-04 (maintainer): + +- ✅ Land the cells that fit now, and skip the three overcommitted cells + until group-drops lands. This temporarily revises the original Goal's + rule against loosening what the benchmark measures, and the revision + lasts until those cells run again. +- Not chosen: keep the whole benchmark waiting on group-drops. + +Landed in [moq-dev/moq#7838](https://github.com/moq-dev/moq/pull/7838): + +- `Cell::fits`: a cell's incomplete units fit when `streams × min(frame, + 1 MiB storage segment)` is at most the connection window. + Equal-priority tracks interleave by packet, so each can hold one frame, + or one segment of a larger frame, before any completes. The default, + 1-stream, 16 KiB-frame, and 8/32 MiB-window cells fit. They pass on + both paths with every frame delivered and nothing evicted. +- The CSV adds `delivered_frames` and `evicted_streams`. + `sh/gh/bench-report.py` reads them by exact name as measurements, not + dimensions. +- `measure` already counts an `Evicted` group as lost when a cell + doesn't fit, and it asserts zero evictions only for cells that fit. +- [moq-dev/moq#7729](https://github.com/moq-dev/moq/pull/7729)'s zero + default max age expired the benchmark's superseded in-flight groups at + the publisher. The benchmark's tracks now set no max age. + +What remains: + +- Each track paces itself on the groups its reader resolves. The + subscriber also abandons some groups without `recv_group` ever + returning them: a stream evicted before its group header was + identified, or an evicted group whose refill FETCH was evicted too + (traced as `fetch abandoned`). Such a group's in-flight permit never + returns, so the writer stalls and the reader times out. The 4 MiB-frame + cell hit this on every run, and the 256-stream cell intermittently. +- Once group-drops lands, the reader returns a dropped group's permit and + counts the group as lost. Then remove the skip (`OVERCOMMITTED` and the + `Cell::fits` partition in `benchmark_receive_windows`), and run the full + sweep through `quest gate`. +- Retain the stream/frame/window axes and the actual offered workload + for [Receive pool](/quest/m2/receive-pool.md). +- Repro: `just rs test -p moq-tokio --test integration --release -E + 'test(benchmark_receive_windows)' --run-ignored only --success-output + immediate`. + +Investigation history: the regression boundary at +[moq-dev/moq#7624](https://github.com/moq-dev/moq/pull/7624), where the +stock connection window shrank from 128 MiB to 4 MiB, and the rejected +connection-wide admission semaphore are recorded in +[moq-dev/moq#7838](https://github.com/moq-dev/moq/pull/7838). + Public API: none. Wire: none. ## Waiting -- Outside condition: the m1 quest `quest/m1/group-drops.md` (planned in - [moq-dev/moq#7890](https://github.com/moq-dev/moq/pull/7890)) finishes, - so `track::Subscriber`'s drop signal is public on `main`. Check it this - way: #7890 has merged, and that quest file has since been deleted from - `main`. +- Outside condition: `quest/m1/group-drops.md`, planned in + [moq-dev/moq#7890](https://github.com/moq-dev/moq/pull/7890), finishes, + making `track::Subscriber`'s drop signal public on `main`. Check that + [moq-dev/moq#7890](https://github.com/moq-dev/moq/pull/7890) has + merged and that the quest file has since been deleted from `main`. diff --git a/rs/moq-tokio/tests/integration/receive_credit.rs b/rs/moq-tokio/tests/integration/receive_credit.rs index 02c259782bc..b411b5d996a 100644 --- a/rs/moq-tokio/tests/integration/receive_credit.rs +++ b/rs/moq-tokio/tests/integration/receive_credit.rs @@ -926,6 +926,10 @@ const INFLIGHT: usize = 1 << 20; /// The receiver's storage segment (`moq_net`'s crate-private `frame::SEGMENT`): a frame within /// one keeps its credit until it completes, and a larger one returns each segment's as it seals. const SEGMENT: usize = 1 << 20; +/// Why cells whose incomplete units overcommit the window are skipped: the quest that makes +/// `track::Subscriber` report each group it abandons, which their readers need to return its +/// in-flight credit. +const OVERCOMMITTED: &str = "waits on quest/m1/group-drops.md"; /// One benchmark configuration. #[derive(Clone, Copy)] @@ -1078,10 +1082,14 @@ async fn measure(cell: Cell, profile: Option, bytes: usize) /// stream window that follows the connection window. Reports the peak credit the peer had /// spent (leased plus unread), throughput of delivered frames, their completion latency, how /// many frames were delivered and streams evicted, and the bottleneck's queue drops, each the -/// median of the runs. Cells whose incomplete units overcommit the window keep their offered -/// load and report what eviction cost them; any other cell fails if a run evicted a stream. -/// The publisher is a stock server, so its send window bounds what is in flight as it would in -/// production. Run by nightly. +/// median of the runs. Any cell whose incomplete units [fit](Cell::fits) fails if a run +/// evicted a stream. The publisher is a stock server, so its send window bounds what is in +/// flight as it would in production. Run by nightly. +/// +/// Cells whose units overcommit the window are skipped, each printed with [`OVERCOMMITTED`] +/// as the reason. They should keep their offered load and report what eviction cost them, but +/// the subscriber can abandon a group without handing it out, so its in-flight credit never +/// returns and the track's writer stalls. #[tokio::test(flavor = "multi_thread")] #[ignore = "release benchmark, run by nightly"] async fn benchmark_receive_windows() { @@ -1115,6 +1123,17 @@ async fn benchmark_receive_windows() { follow: true, ..default })); + let (cells, skipped): (Vec<_>, Vec<_>) = cells.partition(Cell::fits); + for cell in skipped { + // Not stdout: the nightly report reads every line after the table's header as a row. + eprintln!( + "skipped {} cell, {} streams of {} B frames over a {} MiB window: {OVERCOMMITTED}", + cell.axis, + cell.streams, + cell.frame, + cell.window >> 20, + ); + } let paths = [ ("loopback", None, BENCH_BYTES), ("bufferbloat", Some(bufferbloat(BENCH_BLOAT.0)), BENCH_BLOAT.1), @@ -1122,7 +1141,10 @@ async fn benchmark_receive_windows() { println!( "path,axis,streams,frame,window_mib,stream_window,peak_bytes,mib_per_s,p50_ms,p99_ms,delivered_frames,evicted_streams,overflowed_packets" ); - for (cell, (path, profile, bytes)) in cells.flat_map(|cell| paths.clone().map(|path| (cell, path))) { + for (cell, (path, profile, bytes)) in cells + .into_iter() + .flat_map(|cell| paths.clone().map(|path| (cell, path))) + { let mut runs = Vec::with_capacity(RUNS); for _ in 0..RUNS { runs.push(measure(cell, profile.clone(), bytes).await);