From 962fce909dff74f81815b1612850be869e73c3d8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:06:44 -0800 Subject: [PATCH 01/13] chore(quest): claim quest/m2/perf/uring-quiescence.md Quest-Claim: 04e51600dc5256cd From cd9b7fe3faddb0ea7c8d6620f72e46f3a9047788 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:28:09 -0800 Subject: [PATCH 02/13] perf(moq-uring): run a turn's tasks to quiescence before submitting Co-Authored-By: Claude Opus 5.5 --- rs/kio/src/task.rs | 241 ++++++++++++++++++--- rs/kio/src/waiter.rs | 6 +- rs/moq-relay/src/internal.rs | 6 + rs/moq-uring/src/metrics.rs | 6 + rs/moq-uring/src/quic/driver/connection.rs | 42 +++- rs/moq-uring/src/shared.rs | 5 + rs/moq-uring/src/worker.rs | 155 +++++++++---- 7 files changed, 382 insertions(+), 79 deletions(-) diff --git a/rs/kio/src/task.rs b/rs/kio/src/task.rs index 6c3e583baf7..9c529934ee7 100644 --- a/rs/kio/src/task.rs +++ b/rs/kio/src/task.rs @@ -40,7 +40,8 @@ //! throughout. use std::{ - cell::{Cell, RefCell}, + cell::RefCell, + num::NonZeroUsize, sync::{ Arc, Mutex, atomic::{AtomicBool, AtomicU64, Ordering}, @@ -146,13 +147,17 @@ impl Shared { } std::thread_local! { - static POLLING: Polling = const { Polling { current: Cell::new(None), held: RefCell::new(Vec::new()) } }; + static POLLING: Polling = const { Polling { open: RefCell::new(Vec::new()), held: RefCell::new(Vec::new()) } }; } /// A waker's identity: what [`Waker::will_wake`] compares. type Identity = (*const (), *const RawWakerVTable); -/// The [`Waiter::batch`] running on this thread, and the wakes held for it. +fn identity(waker: &Waker) -> Identity { + (waker.data(), waker.vtable()) +} + +/// The [`Waiter::batch`]es running on this thread, and the wakes held for them. /// /// A set woken from inside its owner's poll (by time that poll advanced, say) /// would wake a task that is already running, and the executor would poll it @@ -160,32 +165,42 @@ type Identity = (*const (), *const RawWakerVTable); /// wake. So the wake is held until the batch returns, then delivered only if the /// set is still pending. A wake from another thread is never held: it cannot /// know where the owner's poll is. +/// +/// Batches nest: a task polled inside its executor's batch may open its own. A +/// wake is held by the innermost open batch it names, so a task's batch does not +/// let its wakes of the executor's set through. pub(crate) struct Polling { - /// The innermost batch's waker; the outer ones wait on their stack frames. - current: Cell>, - /// Held wakes. Each batch owns the ones past the length it found on entry. - held: RefCell>>, + /// The open batches' wakers, innermost last. + open: RefCell>, + /// Held wakes, each with the batch holding it. A batch owns the entries past + /// the length it found on entry that name it; the others belong to a batch + /// around it. + held: RefCell)>>, } impl Polling { /// Run `poll` as the task `waker` wakes, delivering the wakes it held when it returns. pub(crate) fn run(waker: &Waker, poll: impl FnOnce() -> R) -> R { struct Scope { - outer: Option, + identity: Identity, start: usize, } impl Drop for Scope { fn drop(&mut self) { - let held: Vec<_> = POLLING.with(|polling| { - polling.current.set(self.outer); + let mut mine = Vec::new(); + POLLING.with(|polling| { + polling.open.borrow_mut().pop(); let mut held = polling.held.borrow_mut(); - match held.len() > self.start { - true => held.drain(self.start..).collect(), - false => Vec::new(), + let mut index = self.start; + while index < held.len() { + match held[index].0 == self.identity { + true => mine.push(held.remove(index).1), + false => index += 1, + } } }); // Outside the borrow: a waker may poll inline and open a batch of its own. - for shared in held { + for shared in mine { shared.release(); } } @@ -193,26 +208,31 @@ impl Polling { // Delivered on unwind too, so a panicking poll loses no wakeup. Polled from a // thread-local's destructor after the scope is gone, nothing is held at all. + let identity = identity(waker); let _scope = POLLING - .try_with(|polling| Scope { - outer: polling.current.replace(Some((waker.data(), waker.vtable()))), - start: polling.held.borrow().len(), + .try_with(|polling| { + polling.open.borrow_mut().push(identity); + Scope { + identity, + start: polling.held.borrow().len(), + } }) .ok(); poll() } - /// Keep back a wake of `parent` when it is the innermost batch on this thread. + /// Keep back a wake of `parent` when a batch open on this thread runs as it. /// /// A wake from a thread-local's destructor may run after the scope is gone; it /// is delivered. fn hold(parent: &Waker, shared: &Arc) -> bool { POLLING .try_with(|polling| { - if polling.current.get() != Some((parent.data(), parent.vtable())) { + let identity = identity(parent); + if !polling.open.borrow().contains(&identity) { return false; } - polling.held.borrow_mut().push(shared.clone()); + polling.held.borrow_mut().push((identity, shared.clone())); true }) .unwrap_or(false) @@ -379,6 +399,23 @@ impl> Tasks { pub fn poll(&mut self, waiter: &Waiter) -> Poll<()> { self.poll_each(waiter, |_, ()| {}) } + + /// [`poll`](Self::poll) again while a pass woke a task, up to `budget` passes, + /// returning how many ran. + /// + /// A wake chain no deeper than the budget completes in this one call: a task + /// woken by another (or pushed) during a pass runs in the next pass. Each pass + /// still snapshots before running anything, so a chain deeper than the budget, + /// or a task that wakes itself every poll, takes `budget` steps and leaves the + /// rest to the next call, waking the owner as [`poll`](Self::poll) does. + /// + /// For an owner that runs its set until it is quiet before blocking, such as + /// an executor's turn. Run the call inside the owner's [`Waiter::batch`], or + /// the wakes that a later pass takes still wake the owner. + #[cfg_attr(feature = "sim", track_caller)] + pub fn poll_passes(&mut self, waiter: &Waiter, budget: NonZeroUsize) -> usize { + self.passes(waiter, budget, &mut |_, ()| {}) + } } impl Tasks { @@ -386,6 +423,16 @@ impl Tasks { /// each task that returned `Ready`, with its output, as it retires. #[cfg_attr(feature = "sim", track_caller)] pub fn poll_each(&mut self, waiter: &Waiter, mut done: impl FnMut(T, T::Output)) -> Poll<()> { + self.passes(waiter, NonZeroUsize::MIN, &mut done); + match self.len { + 0 => crate::coop::ready(()), + _ => Poll::Pending, + } + } + + /// Run passes until one wakes nothing or `budget` have run; returns how many ran. + #[cfg_attr(feature = "sim", track_caller)] + fn passes(&mut self, waiter: &Waiter, budget: NonZeroUsize, done: &mut impl FnMut(T, T::Output)) -> usize { { let mut parent = self.shared.parent.lock().unwrap(); let waker = waiter.waker(); @@ -393,11 +440,24 @@ impl Tasks { *parent = Some(waker.clone()); } } - // Re-arm before sweeping: a wake landing mid-pass must nudge the owner - // again, since its word will already have been snapshotted. - self.shared.pending.store(false, Ordering::Release); + let mut ran = 0; + loop { + // Re-arm before sweeping: a wake landing mid-pass must nudge the owner + // again, since its word will already have been snapshotted. The same + // flag tells the loop whether the pass woke anything. + self.shared.pending.store(false, Ordering::Release); + self.pass(done); + ran += 1; + if ran == budget.get() || !self.shared.pending.load(Ordering::Acquire) { + return ran; + } + } + } - // One snapshot per owner poll, taken in full before any task runs: + /// Run every task woken before the pass began. + #[cfg_attr(feature = "sim", track_caller)] + fn pass(&mut self, done: &mut impl FnMut(T, T::Output)) { + // One snapshot per pass, taken in full before any task runs: // `swap(0)` takes the woken bits and clears them, so a task woken // during the pass (by another task or a self-wake) lands in the next // pass wherever its slot sits, never this one. Dispatching straight off @@ -459,11 +519,6 @@ impl Tasks { } } self.scratch = scratch; - - match self.len { - 0 => crate::coop::ready(()), - _ => Poll::Pending, - } } } @@ -740,6 +795,88 @@ mod tests { drop(fillers); } + /// A pass budget runs a forward wake chain up to the budget in one call and + /// no further: each pass still snapshots before running anything, so the + /// step past the budget lands in the next call and wakes the owner. + #[test] + fn a_pass_budget_bounds_a_wake_chain() { + type Gates = Vec>; + + /// A chain of parked tasks a word apart, each opening the next gate when its + /// own opens, with the first gate open. + fn chain(tasks: &mut Tasks Poll<()>>>, links: usize) -> (Gates, Vec>) { + let gates: Gates = (0..=links).map(|_| crate::Producer::new(false)).collect(); + let mut polls = Vec::new(); + for link in 0..links { + let consumer = gates[link].consume(); + let next = gates[link + 1].clone(); + let count = Arc::new(AtomicUsize::new(0)); + polls.push(count.clone()); + tasks.push(Box::new(move |waiter: &Waiter| { + count.fetch_add(1, Ordering::SeqCst); + match consumer.poll(waiter, |open| match **open { + true => Poll::Ready(()), + false => Poll::Pending, + }) { + Poll::Ready(_) => { + open(&next); + Poll::Ready(()) + } + Poll::Pending => Poll::Pending, + } + })); + // Fillers so each link sits in a later word than the one before. + for _ in 1..WORD_BITS { + tasks.push(Box::new(|_: &Waiter| Poll::Pending)); + } + } + assert!(tasks.poll(&Waiter::noop()).is_pending(), "every link parks"); + open(&gates[0]); + (gates, polls) + } + + let wakes = Arc::new(Wakes(AtomicUsize::new(0))); + let waiter = Waiter::new(Waker::from(wakes.clone())); + let budget = NonZeroUsize::new(3).unwrap(); + + // Within the budget: the whole chain runs in one call, which then finds + // nothing left to run. + let mut tasks = Tasks::new(); + let (_gates, polls) = chain(&mut tasks, 2); + assert_eq!(tasks.poll_passes(&waiter, budget), 2, "two links, then quiet"); + let counts: Vec<_> = polls.iter().map(|count| count.load(Ordering::SeqCst)).collect(); + assert_eq!(counts, [2, 2], "the woken link ran in the same call"); + + // Past it: the fourth link waits for the next call, and the owner is told. + let mut tasks = Tasks::new(); + let (_gates, polls) = chain(&mut tasks, 4); + wakes.0.store(0, Ordering::SeqCst); + assert_eq!(tasks.poll_passes(&waiter, budget), 3, "the budget"); + let counts: Vec<_> = polls.iter().map(|count| count.load(Ordering::SeqCst)).collect(); + assert_eq!(counts, [2, 2, 2, 1], "three steps of the chain, no more"); + assert!(wakes.0.load(Ordering::SeqCst) > 0, "the step left over woke the owner"); + assert_eq!(tasks.poll_passes(&waiter, budget), 1, "the last step, then quiet"); + assert_eq!(polls[3].load(Ordering::SeqCst), 2); + } + + /// A task waking itself on every poll runs once per pass, so a budget bounds + /// it rather than letting it spin the call. + #[test] + fn a_self_waking_task_spends_the_budget_and_stops() { + let mut tasks = Tasks::new(); + let polls = Arc::new(AtomicUsize::new(0)); + let counted = polls.clone(); + tasks.push(move |waiter: &Waiter| { + counted.fetch_add(1, Ordering::SeqCst); + waiter.waker().wake_by_ref(); + Poll::Pending + }); + + let waiter = Waiter::noop(); + assert_eq!(tasks.poll_passes(&waiter, NonZeroUsize::new(4).unwrap()), 4); + assert_eq!(polls.load(Ordering::SeqCst), 4, "once per pass"); + } + /// A retired occupant's leftover waker firing into a reused slot costs at /// most one spurious poll of the new occupant, never a double poll: the /// slot's single bit coalesces the stale wake with any real one. @@ -810,6 +947,52 @@ mod tests { assert_eq!(wakes.0.load(Ordering::SeqCst), 1, "the teardown wake was lost"); } + /// A task running a batch of its own inside its owner's batch still has its + /// wakes of the owner's set held for the owner: a later pass that takes them + /// leaves the owner unwoken, and one the budget leaves over wakes it on return. + #[test] + fn a_nested_batch_holds_wakes_for_the_outer_batch() { + type Boxed = Box Poll<()>>; + + for (budget, b_ran, owner_wakes) in [(2, 1, 0), (1, 0, 1)] { + let mut tasks: Tasks = Tasks::new(); + let a_gate = crate::Producer::new(false); + let a_consumer = a_gate.consume(); + let b_gate = crate::Producer::new(false); + let b_opener = b_gate.clone(); + // When its gate opens, A opens B's from inside a batch of its own, the + // way a session driver polls its children. + tasks.push(Box::new(move |waiter: &Waiter| { + waiter.batch(|| { + match a_consumer.poll(waiter, |open| match **open { + true => Poll::Ready(()), + false => Poll::Pending, + }) { + Poll::Ready(_) => { + open(&b_opener); + Poll::Ready(()) + } + Poll::Pending => Poll::Pending, + } + }) + })); + let b_polls = Arc::new(AtomicUsize::new(0)); + tasks.push(Box::new(gated(b_gate.consume(), b_polls.clone()))); + + let wakes = Arc::new(Wakes(AtomicUsize::new(0))); + let waiter = Waiter::new(Waker::from(wakes.clone())); + assert!(tasks.poll(&waiter).is_pending(), "both park"); + open(&a_gate); + wakes.0.store(0, Ordering::SeqCst); + + let budget = NonZeroUsize::new(budget).unwrap(); + let ran = waiter.batch(|| tasks.poll_passes(&waiter, budget)); + assert_eq!(ran, budget.get()); + assert_eq!(b_polls.load(Ordering::SeqCst), 1 + b_ran, "budget {budget}"); + assert_eq!(wakes.0.load(Ordering::SeqCst), owner_wakes, "budget {budget}"); + } + } + /// An owner whose poll wakes a task in its own set, then polls the set, took that /// wake in the same pass: the executor must not be asked to poll it again. #[test] diff --git a/rs/kio/src/waiter.rs b/rs/kio/src/waiter.rs index 1faf324ee90..c8f4b39e363 100644 --- a/rs/kio/src/waiter.rs +++ b/rs/kio/src/waiter.rs @@ -146,9 +146,9 @@ impl Waiter { /// the task until `poll` returns, and drops it if `poll` polled the set since. /// Otherwise the executor polls the task again for a wake its own poll took. /// - /// Use it once, around a task's outermost poll. Only the innermost batch holds - /// wakes, so one nested inside (around a [`Tasks`](crate::Tasks) child, say) - /// lets its siblings' wakes of the owner through, and costs polls. + /// Use it around a task's outermost poll. Batches nest: a child task polled + /// inside may open its own, and a wake of this waiter's sets from inside the + /// child's batch is still held for this one. pub fn batch(&self, poll: impl FnOnce() -> R) -> R { crate::task::Polling::run(&self.waker, poll) } diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index 27918fa437d..ff9c93cc869 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -920,6 +920,11 @@ fn render_uring(out: &mut String, workers: &[UringWorker]) { "Worker drive-loop turns, including the final poll that completes the root future.", |snap| snap.turns, ); + counter( + "moq_relay_uring_passes_total", + "Worker passes over its woken tasks; over turns, the passes a turn takes to settle.", + |snap| snap.passes, + ); counter( "moq_relay_uring_sq_full_enters_total", "io_uring_enter calls forced by a full submission queue.", @@ -1310,6 +1315,7 @@ mod tests { "completions", "enters", "turns", + "passes", "sq_full_enters", "parks", "wakes", diff --git a/rs/moq-uring/src/metrics.rs b/rs/moq-uring/src/metrics.rs index 7299bd91aa1..ae809dccc26 100644 --- a/rs/moq-uring/src/metrics.rs +++ b/rs/moq-uring/src/metrics.rs @@ -104,6 +104,7 @@ pub(crate) struct Counters { pub completions: Counter, pub enters: Counter, pub turns: Counter, + pub passes: Counter, pub sq_full_enters: Counter, sqes: Distribution, cqes: Distribution, @@ -156,6 +157,7 @@ impl Metrics { completions: self.0.completions.get(), enters: self.0.enters.get(), turns: self.0.turns.get(), + passes: self.0.passes.get(), sq_full_enters: self.0.sq_full_enters.get(), sqes: self.0.sqes.snapshot(), cqes: self.0.cqes.snapshot(), @@ -211,6 +213,10 @@ pub struct Snapshot { pub enters: u64, /// Drive-loop turns, including the final poll that completes the root future. pub turns: u64, + /// Passes over the woken tasks. A turn runs passes until one wakes nothing + /// or [`crate::Config::passes`] have run, so this over `turns` is the + /// passes a turn takes, and how close the budget is to binding. + pub passes: u64, /// Enters forced by a full submission queue. pub sq_full_enters: u64, /// Submission entries accepted by each enter, including zero. diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index 1797482ff6d..87478a3fbab 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -23,6 +23,11 @@ pub(crate) type Shared = Rc; /// One GSO train is at most 64 segments; stay a hair under the kernel cap. const TRAIN_SEGMENTS: usize = 63; +/// The GSO trains a connection stages per worker turn before it yields to +/// the next, so one connection's backlog shares the socket with the others +/// and with the completions the next reap brings back. +const TRAINS_PER_TURN: usize = 1; + /// Everything the handles and the driver share, single-threaded. /// /// The connection and the bookkeeping live in separate `RefCell`s so a stream @@ -424,6 +429,8 @@ pub(crate) fn launch( key, deadline: owner.timer(), close_staged: false, + turn: 0, + trains: 0, }; let future = async move { kio::wait(|waiter| driver.poll(waiter)).await }; (shared, future) @@ -773,6 +780,10 @@ struct Driver { /// flush that came back empty: moq-quic paces the close like any other packet, /// so it can sit behind a pacing timer after the application asked for it. close_staged: bool, + /// The worker turn `trains` counts in. + turn: u64, + /// Trains staged this turn. + trains: usize, } impl Driver { @@ -891,18 +902,29 @@ impl Driver { } } - /// Stage one GSO train, then yield so another connection sharing the - /// socket gets a chance at the transmit pool. + /// Stage up to [`TRAINS_PER_TURN`] GSO trains this turn, then yield the + /// rest to the next turn, so other connections sharing the socket get a + /// chance at the transmit pool and the ring is reaped in between. fn flush(&mut self, waiter: &kio::Waiter) -> Poll { - match self.flush_one(waiter) { - Poll::Ready(Ok(())) => {} - Poll::Ready(Err(err)) => return Poll::Ready(err), - // Backpressure, or nothing left to stage. - Poll::Pending => return Poll::Pending, + let turn = self.shared.owner.turn(); + if self.turn != turn { + self.turn = turn; + self.trains = 0; + } + while self.trains < TRAINS_PER_TURN { + match self.flush_one(waiter) { + Poll::Ready(Ok(())) => self.trains += 1, + Poll::Ready(Err(err)) => return Poll::Ready(err), + // Backpressure, or nothing left to stage. + Poll::Pending => return Poll::Pending, + } + // The budget is spent: requeue behind the next reap, once per turn. + // If moq-quic is drained by then, that poll costs one empty acquire + // and then parks normally. + if self.trains == TRAINS_PER_TURN { + self.shared.owner.yield_turn(waiter.waker()); + } } - // Requeue behind the other ready tasks. If moq-quic is drained, the next - // poll costs one empty acquire and then parks normally. - waiter.waker().wake_by_ref(); Poll::Pending } diff --git a/rs/moq-uring/src/shared.rs b/rs/moq-uring/src/shared.rs index 5c7f9a19209..e91de242c49 100644 --- a/rs/moq-uring/src/shared.rs +++ b/rs/moq-uring/src/shared.rs @@ -61,6 +61,11 @@ pub(crate) struct Shared { pub deadlines: kio::clock::Deadlines, /// Whether a turn is running, so `deadlines` already holds its instant. pub turning: Cell, + /// Counts the worker's turns, so a task can budget work per turn. + pub turn: Cell, + /// Tasks that [yielded](crate::worker::Owner::yield_turn) this turn, woken + /// once its passes are done so they run after the next reap. + pub yields: RefCell>, /// Tasks handed to [`crate::Handle::spawn`], drained by the worker loop. pub spawns: RefCell>, pub unpark: std::sync::Arc, diff --git a/rs/moq-uring/src/worker.rs b/rs/moq-uring/src/worker.rs index e200000c8c6..552ac8d7e83 100644 --- a/rs/moq-uring/src/worker.rs +++ b/rs/moq-uring/src/worker.rs @@ -1,9 +1,11 @@ //! The per-thread worker: ring ownership, the drive loop, and parking. use std::cell::RefCell; +use std::num::NonZeroUsize; use std::rc::{Rc, Weak}; use std::sync::atomic::Ordering; -use std::task::{Context, Poll}; +use std::pin::Pin; +use std::task::{Context, Poll, Waker}; use std::time::Duration; use io_uring::{EnterFlags, IoUring, opcode, types}; @@ -53,11 +55,14 @@ const TEARDOWN_EINTR_RETRIES: usize = 8; /// Maximum time spent staging cancellations and draining completions. const TEARDOWN_TIMEOUT: Duration = Duration::from_millis(3200); +/// The default [`Config::passes`]. +const PASSES: NonZeroUsize = NonZeroUsize::new(4).unwrap(); + /// Worker construction knobs. /// /// The worker sizes its ring internally, with a completion queue that /// comfortably covers the per-socket pool ceilings in [`udp::Config`]. -#[derive(Debug, Default)] +#[derive(Debug)] #[non_exhaustive] pub struct Config { /// Where the worker accumulates its counters. @@ -69,6 +74,23 @@ pub struct Config { /// so give each worker its own. Cloning this config starts a fresh set for /// the cloned worker. pub metrics: Metrics, + /// The most passes over its woken tasks one turn runs before it submits. + /// + /// A turn runs its tasks until a pass wakes nothing, so work a completion + /// sets off (a datagram routed to its connection, the connection staging + /// its reply) is staged by the enter that follows it. The budget bounds a + /// wake chain, or a task waking itself, so the ring is still reaped and + /// submitted to between steps of it. One is a pass per turn. + pub passes: NonZeroUsize, +} + +impl Default for Config { + fn default() -> Self { + Self { + metrics: Metrics::default(), + passes: PASSES, + } + } } impl Clone for Config { @@ -77,6 +99,7 @@ impl Clone for Config { // not be folded into the source worker's per-worker series. Self { metrics: Metrics::default(), + passes: self.passes, } } } @@ -111,6 +134,10 @@ pub struct Worker { owner: kio::Waiter, /// A manual clock's wake for the next deadline; the host clock needs none. alarm: Option, + /// The most passes over the woken tasks a turn runs. + passes: NonZeroUsize, + /// The yields a turn wakes once its passes are done, kept to reuse. + yields: Vec, } impl Worker { @@ -120,7 +147,7 @@ impl Worker { /// park timeout, and batched minimum waits with one code path; there is /// deliberately no fallback (use the tokio stack instead). pub fn new(config: Config) -> Result { - let Config { metrics } = config; + let Config { metrics, passes } = config; let metrics = metrics.counters().clone(); let ring = IoUring::builder() .setup_single_issuer() @@ -166,6 +193,8 @@ impl Worker { clock, deadlines, turning: std::cell::Cell::new(false), + turn: std::cell::Cell::new(0), + yields: RefCell::new(Vec::new()), spawns: RefCell::new(Vec::new()), unpark, metrics, @@ -181,6 +210,8 @@ impl Worker { cqes, futex_armed: false, alarm: None, + passes, + yields: Vec::new(), }) } @@ -203,42 +234,72 @@ impl Worker { pub fn block_on(&mut self, future: F) -> Result { let mut future = std::pin::pin!(future); let waker = self.shared.unpark.waker(); + // A turn runs as one batch of the worker's own poll: a wake of the task + // set from a completion, the root future, or a task is held rather than + // delivered as a wake of the worker. The passes take it, and only one + // the pass budget leaves over wakes the worker for another turn. + let batch = kio::Waiter::new(waker.clone()); let _turn = Turn::enter(&self.shared); loop { - self.shared.metrics.turns.add(1); - // The turn's one clock read: every timer due by now fires, and - // everything polled below sees this instant. - self.shared.deadlines.advance(self.shared.clock.now()); - // Adopt tasks spawned since the last turn (spawning wakes us). - let spawns = std::mem::take(&mut *self.shared.spawns.borrow_mut()); - for task in spawns { - self.tasks.push(task); - } - // A failure recorded between calls (or as the last call's future - // resolved) surfaces before a ready future can return past it. - if let Some(err) = self.shared.failed.take() { - return Err(err.into()); - } - - let cx = Context::from_waker(&waker); - let waiter = self.park.hold(&cx); - if let Poll::Ready(value) = waiter.poll_future(future.as_mut()) { + if let Some(value) = batch.batch(|| self.turn(future.as_mut(), &waker))? { return Ok(value); } - // `Ready` just means the set is drained; the waiter stays - // registered for the next push. - let _ = self.tasks.poll(waiter); + self.finish()?; + } + } - if let Some(err) = self.shared.failed.take() { - return Err(err.into()); - } - self.pump()?; - self.maybe_park()?; + /// One turn up to its submit: completions in, then the root future and the + /// tasks, until a pass wakes nothing or the pass budget is spent. + fn turn(&mut self, future: Pin<&mut F>, waker: &Waker) -> Result, Error> { + self.shared.metrics.turns.add(1); + self.shared.turn.set(self.shared.turn.get() + 1); + // The turn's one clock read: every timer due by now fires, and + // everything polled below sees this instant. + self.shared.deadlines.advance(self.shared.clock.now()); + // Adopt tasks spawned since the last turn (spawning wakes us). + let spawns = std::mem::take(&mut *self.shared.spawns.borrow_mut()); + for task in spawns { + self.tasks.push(task); + } + // Dispatch what the last enter brought back first, so the passes below + // run on it and the sends they stage ride the next enter. + self.reap(None)?; + // A failure recorded between calls (or as the last call's future + // resolved) surfaces before a ready future can return past it. + if let Some(err) = self.shared.failed.take() { + return Err(err.into()); + } + + let cx = Context::from_waker(waker); + let waiter = self.park.hold(&cx); + if let Poll::Ready(value) = waiter.poll_future(future) { + return Ok(Some(value)); + } + // The waiter stays registered for the next push even once the set is + // drained. + let passes = self.tasks.poll_passes(waiter, self.passes); + self.shared.metrics.passes.add(passes as u64); + Ok(None) + } + + /// End a turn: park on its submissions when nothing is left to run, or + /// submit them and turn again. + fn finish(&mut self) -> Result<(), Error> { + // Woken outside the turn's batch, so each wakes the worker: a task that + // yielded runs after the next reap, not in another pass of this turn. + std::mem::swap(&mut self.yields, &mut *self.shared.yields.borrow_mut()); + for waker in self.yields.drain(..) { + waker.wake(); + } + if let Some(err) = self.shared.failed.take() { + return Err(err.into()); } + self.submit_unparked()?; + self.maybe_park() } - /// Dispatch completions, carrying parked submissions into the wait enter. - fn pump(&mut self) -> Result<(), Error> { + /// Submit what cannot ride the parked enter. + fn submit_unparked(&mut self) -> Result<(), Error> { let notified = self.shared.unpark.word.load(Ordering::Acquire) == crate::park::NOTIFIED; let full = !notified && !self.futex_armed && self.shared.ring.borrow_mut().submission().is_full(); if full { @@ -249,7 +310,7 @@ impl Worker { if notified || full { self.submit()?; } - self.reap(None) + Ok(()) } /// Submit and reap completions only while the teardown budget remains. @@ -706,6 +767,22 @@ impl Owner { } } + /// The worker's turn count, so a task can budget its work per turn. Zero + /// once the worker is gone. + #[cfg_attr(not(feature = "quic"), allow(dead_code, reason = "only the `quic` feature uses it"))] + pub fn turn(&self) -> u64 { + self.shared.upgrade().map_or(0, |shared| shared.turn.get()) + } + + /// Wake `waker` once this turn's passes are done, so the task runs again + /// after the next reap rather than in a later pass of this turn. + #[cfg_attr(not(feature = "quic"), allow(dead_code, reason = "only the `quic` feature uses it"))] + pub fn yield_turn(&self, waker: &Waker) { + if let Some(shared) = self.shared.upgrade() { + shared.yields.borrow_mut().push(waker.clone()); + } + } + /// A disarmed timer on the worker. #[cfg_attr(not(feature = "quic"), allow(dead_code, reason = "only the `quic` feature uses it"))] pub fn timer(&self) -> crate::Timer { @@ -852,7 +929,8 @@ mod tests { .expect("stage NOP"); let mut timer = worker.handle().timer(); timer.set(Some(due(&worker))); - worker.pump().expect("pump"); + worker.submit_unparked().expect("submit"); + worker.reap(None).expect("reap"); worker.maybe_park().expect("park"); assert_eq!( worker.handle().metrics().snapshot().enters, @@ -918,7 +996,8 @@ mod tests { .push(&opcode::Nop::new().build().user_data(key)) .expect("stage NOP"); } - worker.pump().expect("full pump"); + worker.submit_unparked().expect("full submit"); + worker.reap(None).expect("reap"); let snapshot = worker.handle().metrics().snapshot(); assert_eq!(snapshot.sq_full_enters, 1); assert_eq!(snapshot.enters, 1); @@ -942,7 +1021,8 @@ mod tests { .push(&opcode::Nop::new().build().user_data(key)) .expect("stage NOP"); } - worker.pump().expect("full pump"); + worker.submit_unparked().expect("full submit"); + worker.reap(None).expect("reap"); worker.maybe_park().expect("park with armed futex"); let after = worker.handle().metrics().snapshot(); assert_eq!(after.enters - before.enters, 1); @@ -974,10 +1054,11 @@ mod tests { std::thread::yield_now(); } assert!(worker.shared.ring.borrow_mut().submission().is_empty()); - // A continuously self-waking future never parks. Its next pump must + // A continuously self-waking future never parks. Its next turn must // dispatch the receive even though there is nothing new to submit. worker.shared.unpark.unpark(); - worker.pump().expect("notified pump"); + worker.submit_unparked().expect("notified submit"); + worker.reap(None).expect("reap"); let Poll::Ready(Ok(packet)) = sock.poll_recv(&kio::Waiter::noop()) else { panic!("one busy turn must publish the receive"); }; From bfe3719dc50330bd3b5c78795933e997687b361b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:46:34 -0800 Subject: [PATCH 03/13] perf(moq-uring): take a turn's stale notifications after its reap Co-Authored-By: Claude Opus 5.5 --- rs/moq-uring/src/worker.rs | 92 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 3 deletions(-) diff --git a/rs/moq-uring/src/worker.rs b/rs/moq-uring/src/worker.rs index 552ac8d7e83..5e29e0fd624 100644 --- a/rs/moq-uring/src/worker.rs +++ b/rs/moq-uring/src/worker.rs @@ -256,14 +256,19 @@ impl Worker { // The turn's one clock read: every timer due by now fires, and // everything polled below sees this instant. self.shared.deadlines.advance(self.shared.clock.now()); + // Dispatch what the last enter brought back first, so the passes below + // run on it and the sends they stage ride the next enter. + self.reap(None)?; + // Everything below polls whatever a wake so far could have readied, so + // take the notifications that came before it, and leave only the ones + // raised from here on to keep the turn from parking. The swap acquires + // a remote wake's store, so the task bits it set before it are seen. + self.shared.unpark.word.swap(RUNNING, Ordering::AcqRel); // Adopt tasks spawned since the last turn (spawning wakes us). let spawns = std::mem::take(&mut *self.shared.spawns.borrow_mut()); for task in spawns { self.tasks.push(task); } - // Dispatch what the last enter brought back first, so the passes below - // run on it and the sends they stage ride the next enter. - self.reap(None)?; // A failure recorded between calls (or as the last call's future // resolved) surfaces before a ready future can return past it. if let Some(err) = self.shared.failed.take() { @@ -1065,6 +1070,87 @@ mod tests { assert_eq!(packet.payload(), b"deferred"); } + /// A completion's wake chain runs in the turn that reaps it: a receive task + /// hands each request to a sending task, so with a pass for each hop the + /// reply is staged before the turn submits, and rides its parked enter. + #[test] + fn a_reply_is_staged_in_the_turn_that_reaps_its_request() { + const ROUNDS: u64 = 16; + + /// Turns and enters over `ROUNDS` round trips under a pass budget. + fn per_round(passes: usize) -> Option<(u64, u64)> { + let mut config = Config::default(); + config.passes = NonZeroUsize::new(passes).unwrap(); + let mut worker = worker_with(config)?; + let handle = worker.handle(); + let mut udp_config = udp::Config::default(); + udp_config.gro = false; + udp_config.gso = false; + let bind = || std::net::UdpSocket::bind("127.0.0.1:0").expect("bind"); + let client = handle.udp(bind(), udp_config.clone()).expect("client"); + let server = Rc::new(handle.udp(bind(), udp_config).expect("server")); + let (client_addr, server_addr) = (client.local_addr().unwrap(), server.local_addr().unwrap()); + + let requests = kio::Producer::new(0u64); + let receiver = server.clone(); + let received = requests.clone(); + handle.spawn(async move { + while receiver.recv().await.is_ok() { + let Ok(mut count) = received.write() else { return }; + *count += 1; + } + }); + let sender = server.clone(); + let consumer = requests.consume(); + handle.spawn(async move { + let mut replied = 0; + loop { + let next = kio::wait(|waiter: &kio::Waiter| { + match consumer.poll(waiter, |count| match **count > replied { + true => Poll::Ready(**count), + false => Poll::Pending, + }) { + Poll::Ready(Ok(count)) => Poll::Ready(count), + Poll::Ready(Err(_)) => Poll::Ready(replied), + Poll::Pending => Poll::Pending, + } + }) + .await; + if next == replied { + return; + } + replied = next; + let mut tx = sender.acquire().await.expect("send buffer"); + tx.extend_from_slice(b"pong"); + tx.send(udp::Transmit::new(client_addr, 4)).expect("reply"); + } + }); + + let (before, after) = worker + .block_on(async { + let round = || async { + let mut tx = client.acquire().await.expect("send buffer"); + tx.extend_from_slice(b"ping"); + tx.send(udp::Transmit::new(server_addr, 4)).expect("request"); + assert_eq!(client.recv().await.expect("reply").payload(), b"pong"); + }; + round().await; + let before = handle.metrics().snapshot(); + for _ in 0..ROUNDS { + round().await; + } + (before, handle.metrics().snapshot()) + }) + .expect("worker"); + Some((after.turns - before.turns, after.enters - before.enters)) + } + + let Some(one) = per_round(1) else { return }; + let Some(two) = per_round(2) else { return }; + assert_eq!(two, (2 * ROUNDS, 2 * ROUNDS), "each hop is one turn and one enter"); + assert_eq!(one, (3 * ROUNDS, 2 * ROUNDS), "a pass per turn spends a turn on the hand-off"); + } + #[test] fn ready_future() { let Some(mut worker) = worker() else { return }; From 08ab7d698e83e9bbbfb799c19c59ff381a8ee52a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:54:00 -0800 Subject: [PATCH 04/13] perf(moq-uring): stage four GSO trains per connection per turn Co-Authored-By: Claude Opus 5.5 --- rs/moq-uring/README.md | 10 ++++++++-- rs/moq-uring/src/quic/driver/connection.rs | 6 ++++-- rs/moq-uring/src/worker.rs | 4 +++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/rs/moq-uring/README.md b/rs/moq-uring/README.md index bd47fb53c30..1f016664d68 100644 --- a/rs/moq-uring/README.md +++ b/rs/moq-uring/README.md @@ -19,6 +19,12 @@ local (`!Send`) task set, and the UDP sockets bound through it. turn, from its one clock read; the earliest deadline rides `io_uring_enter` as an absolute timeout. Zero timeout SQEs. The queue is the worker's ambient clock, and `Handle::run` drives MoQ with it and a single timer. +- **Turns**: each turn reaps and dispatches completions first, then polls the + root future and runs passes over the woken tasks until a pass wakes nothing + or `Config::passes` have run, then parks or submits. So a received + datagram's routing, its connection's processing, and the reply that stages + are carried by the one enter that follows. A connection stages a bounded + number of GSO trains per turn, then yields its backlog to the next turn. - **Parking**: a futex word per worker. Remote wakes are an atomic store, plus one `futex(2)` wake only while the worker is actually parked (a `FUTEX_WAIT` SQE armed on the word). @@ -48,8 +54,8 @@ local (`!Send`) task set, and the UDP sockets bound through it. thread through `metrics::Metrics::snapshot`. Buffer-pool health (`ENOBUFS`, provided-buffer exhaustion, TX-pool stalls), batch effectiveness (datagrams per receive and per send), ring traffic (submissions, completions, - `io_uring_enter` calls), and scheduling (turns, parks, remote futex wakes, - timer churn). A turn that parks carries its staged SQEs in that enter; + `io_uring_enter` calls), and scheduling (turns, task passes, parks, remote + futex wakes, timer churn). A turn that parks carries its staged SQEs in that enter; notified turns enter only for staged SQEs, deferred completion work, or CQ overflow. `sq_full_enters` counts queue-pressure enters. `sqes` and `cqes` are fixed histograms of submissions accepted and completions made visible diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index 87478a3fbab..8afd8d623ed 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -25,8 +25,10 @@ const TRAIN_SEGMENTS: usize = 63; /// The GSO trains a connection stages per worker turn before it yields to /// the next, so one connection's backlog shares the socket with the others -/// and with the completions the next reap brings back. -const TRAINS_PER_TURN: usize = 1; +/// and with the completions the next reap brings back. More than one, so a +/// reply's lone train is followed by an empty flush rather than a yield and a +/// turn of its own; four cut a bulk sender's enters by a quarter against one. +const TRAINS_PER_TURN: usize = 4; /// Everything the handles and the driver share, single-threaded. /// diff --git a/rs/moq-uring/src/worker.rs b/rs/moq-uring/src/worker.rs index 5e29e0fd624..6a07c754ed2 100644 --- a/rs/moq-uring/src/worker.rs +++ b/rs/moq-uring/src/worker.rs @@ -55,7 +55,9 @@ const TEARDOWN_EINTR_RETRIES: usize = 8; /// Maximum time spent staging cancellations and draining completions. const TEARDOWN_TIMEOUT: Duration = Duration::from_millis(3200); -/// The default [`Config::passes`]. +/// The default [`Config::passes`]. A received datagram's chain is the demux, +/// its connection, the application task it wakes, and the connection again +/// for the reply; more found nothing left to run. const PASSES: NonZeroUsize = NonZeroUsize::new(4).unwrap(); /// Worker construction knobs. From f15bd0951c6cc260733046560766cdd03e739e75 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:55:29 -0800 Subject: [PATCH 05/13] quest: finish run to quiescence Co-Authored-By: Claude Opus 5.5 --- quest/m2/perf/README.md | 7 +-- quest/m2/perf/uring-quiescence.md | 62 -------------------- quest/m2/performance-survey/task-capacity.md | 8 +-- 3 files changed, 6 insertions(+), 71 deletions(-) delete mode 100644 quest/m2/perf/uring-quiescence.md diff --git a/quest/m2/perf/README.md b/quest/m2/perf/README.md index 23a8199d184..c3fece7e619 100644 --- a/quest/m2/perf/README.md +++ b/quest/m2/perf/README.md @@ -20,10 +20,8 @@ Planning quests can settle their contracts independently. Facts from the 2026-09 hot-path survey, so quests don't re-litigate them: - Decided in the 2026-10-05 audit: perf quests that edit moq-uring's QUIC - driver (Run to quiescence) measure their before and after on `moq-quic`. -- `moq-uring`'s only backend is `moq-quic`. Every profile names its backend. The - historical quiche-flavor numbers cited in - [Run to quiescence](/quest/m2/perf/uring-quiescence.md) are re-measured on `moq-quic`. + driver measure their before and after on `moq-quic`. +- `moq-uring`'s only backend is `moq-quic`. Every profile names its backend. - Cross-thread wakeups are already cheap: one futex word per worker, at most one `futex(FUTEX_WAKE)` per park cycle, wake bursts coalesce through the `kio::Tasks` bitset. No eventfd, no MSG_RING, by design (`SINGLE_ISSUER`). @@ -84,7 +82,6 @@ revisions on the same host and worker count, and profiles with - [Relay allocations](/quest/m2/perf/relay-allocs.md) - fewer allocations per delivered frame, starting with per-stream session clones - [QUIC lock contention](/quest/m2/perf/quic-lock-contention.md) - a measured verdict on the tokio driver's connection-wide mutex and per-packet endpoint-to-connection hop, with levers built only on evidence - [Borrow-free QUIC state](/quest/m2/perf/quic-borrow-free.md) - a prototyped report on sharing connection state with stream handles without `RefCell` or `Mutex`: context passing vs driver-owned state with per-stream channels -- [Run to quiescence](/quest/m2/perf/uring-quiescence.md) - a received packet's reply is staged in the same turn, under a pass and train budget that keeps the fairness rule - [Ingest batch](/quest/m2/perf/ingest-batch.md) - relay ingest pays one lock, wake, and clock read per burst of whole frames instead of per frame - [Streaming frame read owners](/quest/m2/perf/frame-read-owners.md) - readers of a frame still arriving share one byte owner per publication instead of allocating one each per read - [Shard by default](/quest/m2/perf/shard-default.md) - a saturating shard sweep decides whether an unset `runtime.workers` runs one worker per core on Linux diff --git a/quest/m2/perf/uring-quiescence.md b/quest/m2/perf/uring-quiescence.md deleted file mode 100644 index cb29f6bb02e..00000000000 --- a/quest/m2/perf/uring-quiescence.md +++ /dev/null @@ -1,62 +0,0 @@ -# [M] Run to quiescence before submit - -## Goal - -A received packet's QUIC processing and the sends it produces are staged in -the same drive-loop turn, so one enter carries the reply. Today -`Tasks::poll` runs one pass over the ready set per turn and a mid-pass wake -lands in the next turn by design (`Tasks::poll` in rs/kio/src/task.rs, test -`a_mid_pass_wake_lands_in_the_next_pass`), so a datagram costs three turns: -dispatch sets the demux bit, the demux drains packets and kicks each -connection driver (`Endpoint::demux` in -rs/moq-uring/src/quic/driver/endpoint.rs), and the drivers stage `SendMsg` a turn later. Every hop is a turn -and up to an enter. The other io_uring runtimes drain their local queue to -exhaustion, or to a quantum, before touching the ring. - -## Plan - -The pass budget builds on `kio`'s current `Tasks::poll` (#4156 merged -`Pollable` into `Task`). Keep the fairness the one-pass rule protects: a forward wake -chain must not starve the caller's other arms, and one connection's backlog -must not starve the socket. - -- `Tasks::poll` takes a pass budget. It re-snapshots the bitset after each - pass and runs again while anything is set and the budget holds, so a chain - of bounded depth completes within the turn. A chain deeper than the budget - is deferred exactly as today. The budget is a `Worker` setting; sweep 1 - (today), 2, 4, and 8. -- Dispatch runs before the task pass, not after: the loop becomes reap and - dispatch, then tasks to quiescence, then submit or park. That removes the - stale first pass a park-returning turn runs today (the worker loop in - rs/moq-uring/src/worker.rs polls tasks on readiness the previous turn - already consumed). -- The egress driver stages one GSO train of `TRAIN_SEGMENTS = 63` segments - per turn and then wakes itself (`Driver::flush` in - rs/moq-uring/src/quic/driver/connection.rs), so a deep backlog pays a - whole turn per train and would consume every pass. That cadence is a - hardcoded fairness choice across connections sharing a socket. Make it a - trains-per-turn budget on `flush` and sweep 1, 2, and 4 together with the - pass budget, under the fanout and single-heavy-connection shapes, so trains - per turn and passes per turn are measured together. A no-win keeps 1 train. - The #3120 numbers are the deleted quiche driver's; re-profile on `moq-quic`. - Decided in the 2026-09-30 audit: the egress requeue quest merged here, since - both budgets need the same sweep. -- Add `passes` per turn to the metrics beside `turns`. -- [Task capacity](/quest/m2/performance-survey/task-capacity.md) also - reshapes the `Tasks::poll` wake-word snapshot. Whichever of the two lands - second rebases its benchmark on the other. -- Ingress fairness: a pass budget does not bound work inside one poll. - The endpoint's `Inner::poll_run` calls `poll_recv` until `Pending`, so one - poll drains every queued receive completion, and `Endpoint::demux` drains - every datagram of each, where one GRO completion can carry many. Measure - saturated ingress against timers, control messages, and quiet connections - on the same worker, and if their latency shows it, budget both the - completions per poll and the datagrams per turn. Decided in the 2026-10-22 - io_uring audit: a latency question, not a known starvation bug. - -Acceptance: turns and enters per received datagram on the chat shape -(request-reply is the worst case), CPU per Gbps and p99 latency on the fanout -shape, on Linux with the io_uring workers at both base and current, using -`just bench-runtime` at each revision with the same worker count. The existing -starvation test keeps its meaning with a budget of 1 and gains a sibling -proving the budget bound. diff --git a/quest/m2/performance-survey/task-capacity.md b/quest/m2/performance-survey/task-capacity.md index 708e03195dd..ccdcb0295b7 100644 --- a/quest/m2/performance-survey/task-capacity.md +++ b/quest/m2/performance-survey/task-capacity.md @@ -48,10 +48,10 @@ Candidates to evaluate: reclaimable slot chunks, and an index of nonempty ready words so a poll does not scan historical capacity. Include owner polls triggered by unrelated active work in the sweep. -Related: [worker quiescence](/quest/m2/perf/uring-quiescence.md) and -[relay allocations](/quest/m2/perf/relay-allocs.md) own other costs. -Worker quiescence also reshapes the `Tasks::poll` snapshot with a pass -budget; whichever of the two lands second rebases its benchmark on the other. +Related: [relay allocations](/quest/m2/perf/relay-allocs.md) owns other costs. +The io_uring worker now polls its set through `Tasks::poll_passes`, which runs +the per-pass snapshot up to a pass budget per turn, so a snapshot's cost is +paid once per pass, not once per turn. Follow the [survey's benchmark contract](/quest/m2/performance-survey/README.md): measure the baseline before optimizing, retain paired evidence, verify equivalent From b6c5f8a79035efe69eb07be2c99b17a4c6924091 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 02:22:09 -0800 Subject: [PATCH 06/13] style: rustfmt Co-Authored-By: Claude Opus 5.5 --- rs/kio/src/task.rs | 5 ++++- rs/moq-uring/src/worker.rs | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/rs/kio/src/task.rs b/rs/kio/src/task.rs index 9c529934ee7..5c06a7900b1 100644 --- a/rs/kio/src/task.rs +++ b/rs/kio/src/task.rs @@ -804,7 +804,10 @@ mod tests { /// A chain of parked tasks a word apart, each opening the next gate when its /// own opens, with the first gate open. - fn chain(tasks: &mut Tasks Poll<()>>>, links: usize) -> (Gates, Vec>) { + fn chain( + tasks: &mut Tasks Poll<()>>>, + links: usize, + ) -> (Gates, Vec>) { let gates: Gates = (0..=links).map(|_| crate::Producer::new(false)).collect(); let mut polls = Vec::new(); for link in 0..links { diff --git a/rs/moq-uring/src/worker.rs b/rs/moq-uring/src/worker.rs index 6a07c754ed2..1ce7c52bcb0 100644 --- a/rs/moq-uring/src/worker.rs +++ b/rs/moq-uring/src/worker.rs @@ -2,9 +2,9 @@ use std::cell::RefCell; use std::num::NonZeroUsize; +use std::pin::Pin; use std::rc::{Rc, Weak}; use std::sync::atomic::Ordering; -use std::pin::Pin; use std::task::{Context, Poll, Waker}; use std::time::Duration; @@ -1150,7 +1150,11 @@ mod tests { let Some(one) = per_round(1) else { return }; let Some(two) = per_round(2) else { return }; assert_eq!(two, (2 * ROUNDS, 2 * ROUNDS), "each hop is one turn and one enter"); - assert_eq!(one, (3 * ROUNDS, 2 * ROUNDS), "a pass per turn spends a turn on the hand-off"); + assert_eq!( + one, + (3 * ROUNDS, 2 * ROUNDS), + "a pass per turn spends a turn on the hand-off" + ); } #[test] From e9a5fe7c42d79114e16e787da12eb06cba4742a1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 02:23:37 -0800 Subject: [PATCH 07/13] test(kio): name the chain test's set type Co-Authored-By: Claude Opus 5.5 --- rs/kio/src/task.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/rs/kio/src/task.rs b/rs/kio/src/task.rs index 5c06a7900b1..e0759614341 100644 --- a/rs/kio/src/task.rs +++ b/rs/kio/src/task.rs @@ -801,13 +801,11 @@ mod tests { #[test] fn a_pass_budget_bounds_a_wake_chain() { type Gates = Vec>; + type Set = Tasks Poll<()>>>; /// A chain of parked tasks a word apart, each opening the next gate when its /// own opens, with the first gate open. - fn chain( - tasks: &mut Tasks Poll<()>>>, - links: usize, - ) -> (Gates, Vec>) { + fn chain(tasks: &mut Set, links: usize) -> (Gates, Vec>) { let gates: Gates = (0..=links).map(|_| crate::Producer::new(false)).collect(); let mut polls = Vec::new(); for link in 0..links { From 4ba8636b0caa189a31437f4c1d6bbcbe2b3cfc6f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 02:28:26 -0800 Subject: [PATCH 08/13] test(moq-uring): build the reply test's configs in place Co-Authored-By: Claude Opus 5.5 --- rs/moq-uring/src/worker.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/rs/moq-uring/src/worker.rs b/rs/moq-uring/src/worker.rs index 1ce7c52bcb0..4b5ca01ade9 100644 --- a/rs/moq-uring/src/worker.rs +++ b/rs/moq-uring/src/worker.rs @@ -1081,13 +1081,16 @@ mod tests { /// Turns and enters over `ROUNDS` round trips under a pass budget. fn per_round(passes: usize) -> Option<(u64, u64)> { - let mut config = Config::default(); - config.passes = NonZeroUsize::new(passes).unwrap(); - let mut worker = worker_with(config)?; + let mut worker = worker_with(Config { + passes: NonZeroUsize::new(passes).unwrap(), + ..Config::default() + })?; let handle = worker.handle(); - let mut udp_config = udp::Config::default(); - udp_config.gro = false; - udp_config.gso = false; + let udp_config = udp::Config { + gro: false, + gso: false, + ..udp::Config::default() + }; let bind = || std::net::UdpSocket::bind("127.0.0.1:0").expect("bind"); let client = handle.udp(bind(), udp_config.clone()).expect("client"); let server = Rc::new(handle.udp(bind(), udp_config).expect("server")); From 3ae1a65971a2850d012e2d985024f683eb0ca790 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 02:34:50 -0800 Subject: [PATCH 09/13] test(moq-uring): check the reply's hand-off by the turns its tasks ran in Co-Authored-By: Claude Opus 5.5 --- rs/moq-uring/src/worker.rs | 68 +++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/rs/moq-uring/src/worker.rs b/rs/moq-uring/src/worker.rs index 4b5ca01ade9..ea4b6865ab6 100644 --- a/rs/moq-uring/src/worker.rs +++ b/rs/moq-uring/src/worker.rs @@ -1074,18 +1074,21 @@ mod tests { /// A completion's wake chain runs in the turn that reaps it: a receive task /// hands each request to a sending task, so with a pass for each hop the - /// reply is staged before the turn submits, and rides its parked enter. + /// reply is staged in the turn that reaped its request, before that turn + /// submits. Each task records the worker turn it ran in, which the network's + /// completion order and timing cannot change. #[test] fn a_reply_is_staged_in_the_turn_that_reaps_its_request() { - const ROUNDS: u64 = 16; + const ROUNDS: usize = 16; - /// Turns and enters over `ROUNDS` round trips under a pass budget. - fn per_round(passes: usize) -> Option<(u64, u64)> { + /// For each round trip, the turns from the request's receive to its reply. + fn hand_offs(passes: usize) -> Option> { let mut worker = worker_with(Config { passes: NonZeroUsize::new(passes).unwrap(), ..Config::default() })?; let handle = worker.handle(); + let owner = Owner::new(&handle.shared); let udp_config = udp::Config { gro: false, gso: false, @@ -1096,68 +1099,59 @@ mod tests { let server = Rc::new(handle.udp(bind(), udp_config).expect("server")); let (client_addr, server_addr) = (client.local_addr().unwrap(), server.local_addr().unwrap()); - let requests = kio::Producer::new(0u64); - let receiver = server.clone(); - let received = requests.clone(); + // The turn each request was received in, handed from one task to the other. + let requests = kio::Producer::new(Vec::::new()); + let gaps = Rc::new(RefCell::new(Vec::new())); + let (receiver, received, turns) = (server.clone(), requests.clone(), owner.clone()); handle.spawn(async move { while receiver.recv().await.is_ok() { - let Ok(mut count) = received.write() else { return }; - *count += 1; + let Ok(mut turn) = received.write() else { return }; + turn.push(turns.turn()); } }); - let sender = server.clone(); - let consumer = requests.consume(); + let (sender, consumer, recorded) = (server.clone(), requests.consume(), gaps.clone()); handle.spawn(async move { let mut replied = 0; loop { - let next = kio::wait(|waiter: &kio::Waiter| { - match consumer.poll(waiter, |count| match **count > replied { - true => Poll::Ready(**count), + let Some(received) = kio::wait(|waiter: &kio::Waiter| { + match consumer.poll(waiter, |turns| match turns.len() > replied { + true => Poll::Ready(turns[replied]), false => Poll::Pending, }) { - Poll::Ready(Ok(count)) => Poll::Ready(count), - Poll::Ready(Err(_)) => Poll::Ready(replied), + Poll::Ready(Ok(turn)) => Poll::Ready(Some(turn)), + Poll::Ready(Err(_)) => Poll::Ready(None), Poll::Pending => Poll::Pending, } }) - .await; - if next == replied { + .await + else { return; - } - replied = next; + }; + replied += 1; let mut tx = sender.acquire().await.expect("send buffer"); tx.extend_from_slice(b"pong"); tx.send(udp::Transmit::new(client_addr, 4)).expect("reply"); + recorded.borrow_mut().push(owner.turn() - received); } }); - let (before, after) = worker + worker .block_on(async { - let round = || async { + for _ in 0..ROUNDS { let mut tx = client.acquire().await.expect("send buffer"); tx.extend_from_slice(b"ping"); tx.send(udp::Transmit::new(server_addr, 4)).expect("request"); assert_eq!(client.recv().await.expect("reply").payload(), b"pong"); - }; - round().await; - let before = handle.metrics().snapshot(); - for _ in 0..ROUNDS { - round().await; } - (before, handle.metrics().snapshot()) }) .expect("worker"); - Some((after.turns - before.turns, after.enters - before.enters)) + Some(gaps.take()) } - let Some(one) = per_round(1) else { return }; - let Some(two) = per_round(2) else { return }; - assert_eq!(two, (2 * ROUNDS, 2 * ROUNDS), "each hop is one turn and one enter"); - assert_eq!( - one, - (3 * ROUNDS, 2 * ROUNDS), - "a pass per turn spends a turn on the hand-off" - ); + let Some(two) = hand_offs(2) else { return }; + assert_eq!(two, [0; ROUNDS], "the reply is staged in its request's turn"); + let Some(one) = hand_offs(1) else { return }; + assert_eq!(one, [1; ROUNDS], "a pass per turn spends a turn on the hand-off"); } #[test] From 9aabceecdb2f72ad55bb3081b9599e6098e6492a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 09:32:45 -0800 Subject: [PATCH 10/13] doc(relay): name the uring passes metric Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/http.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/doc/bin/relay/http.md b/doc/bin/relay/http.md index 4386046ba33..3c40c390e98 100644 --- a/doc/bin/relay/http.md +++ b/doc/bin/relay/http.md @@ -98,9 +98,11 @@ With `--runtime-io-uring`, each QUIC worker also reports `moq_relay_uring_*` counters under a `worker` label. Datagrams per enter and the SQE/CQE histograms show whether syscalls carry useful work. Compare turns and queue-pressure enters with throughput: flushing deferred work can remove -empty busy turns, so enters/turn alone does not grade efficiency. The -histograms include empty enters. A dead worker shows stuck zeros rather than -disappearing. +empty busy turns, so enters/turn alone does not grade efficiency. +`moq_relay_uring_passes_total` over turns is how many passes over its woken +tasks a turn takes to settle; a ratio at the default budget of 4 means wake +chains run past it and spill into the next turn. The histograms include empty +enters. A dead worker shows stuck zeros rather than disappearing. ### GET /sessions From b576435b72005a194ea1ef8d459a584c9e0e686f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 08:49:13 -0800 Subject: [PATCH 11/13] test(uring): retain quiescence measurements and busy-task coverage Co-Authored-By: GPT-6.1 Sol --- bench/README.md | 10 +++ doc/bin/relay/http.md | 5 +- rs/kio/src/task.rs | 2 + rs/moq-uring/benches/worker_turns.rs | 117 ++++++++++++++++++++++++++- rs/moq-uring/src/worker.rs | 29 +++++-- 5 files changed, 155 insertions(+), 8 deletions(-) diff --git a/bench/README.md b/bench/README.md index b214288f933..94dac3a0acb 100644 --- a/bench/README.md +++ b/bench/README.md @@ -130,6 +130,16 @@ Runtime comparison requires Linux because io\_uring and relay process metrics come from Linux interfaces. The default worker count is the number of logical CPUs the run may use. +`moq-uring`'s `worker_turns` target also measures QUIC request/reply on one +worker, sweeping 1 or 16 connections against 64 B or 256 KiB replies with +1 or 4 task passes. It prints rounds and counter deltas for turns, passes, +enters and received datagrams, so counts per round can be compared alongside +timing. The existing nightly `rust-uring` group runs these cases. + +```bash +nix develop --command cargo bench --locked -p moq-uring --bench worker_turns -- worker_request_reply +``` + Measure `moq-uring`'s actual `Local` thread guard separately from I/O: ```bash diff --git a/doc/bin/relay/http.md b/doc/bin/relay/http.md index 0e808a52aae..1180dfde437 100644 --- a/doc/bin/relay/http.md +++ b/doc/bin/relay/http.md @@ -101,8 +101,9 @@ queue-pressure enters with throughput: flushing deferred work can remove empty busy turns, so enters/turn alone does not grade efficiency. `moq_relay_uring_passes_total` over turns is how many passes over its woken tasks a turn runs; a ratio near the default budget of 4 shows heavy use of -the pass budget but does not prove that work spilled into the next turn. The histograms include empty -enters. A dead worker shows stuck zeros rather than disappearing. +the pass budget but does not prove that work spilled into the next turn. The +histograms include empty enters. A dead worker shows stuck zeros rather than +disappearing. ### GET /sessions diff --git a/rs/kio/src/task.rs b/rs/kio/src/task.rs index e0b20eb61a3..b6592c15028 100644 --- a/rs/kio/src/task.rs +++ b/rs/kio/src/task.rs @@ -1041,7 +1041,9 @@ mod tests { wakes.0.store(0, Ordering::SeqCst); let budget = NonZeroUsize::new(budget).unwrap(); + let references = Arc::strong_count(&tasks.shared); let ran = waiter.batch(|| tasks.poll_passes(&waiter, budget)); + assert_eq!(Arc::strong_count(&tasks.shared), references, "a batch retained a wake"); assert_eq!(ran, budget.get()); assert_eq!(b_polls.load(Ordering::SeqCst), 1 + b_ran, "budget {budget}"); assert_eq!(wakes.0.load(Ordering::SeqCst), owner_wakes, "budget {budget}"); diff --git a/rs/moq-uring/benches/worker_turns.rs b/rs/moq-uring/benches/worker_turns.rs index 570068b5c9b..66e6fbb0e24 100644 --- a/rs/moq-uring/benches/worker_turns.rs +++ b/rs/moq-uring/benches/worker_turns.rs @@ -2,6 +2,121 @@ use criterion::{criterion_group, criterion_main}; +#[cfg(all(target_os = "linux", feature = "quic"))] +#[path = "../tests/integration/support.rs"] +mod support; + +/// A request and its reply traverse demux, connection and application tasks. +/// Sweep connections and reply bytes so batching cannot hide a fanout cost. +#[cfg(all(target_os = "linux", feature = "quic"))] +#[expect( + clippy::disallowed_methods, + clippy::disallowed_types, + reason = "benchmarks measure themselves on the host clock" +)] +fn request_reply(c: &mut criterion::Criterion) { + use std::{net::UdpSocket, num::NonZeroUsize, time::Instant}; + + use criterion::{BenchmarkId, Throughput}; + use moq_uring::{Config, Worker, quic, udp}; + + const ALPN: &str = "worker-turns"; + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let certs = support::certs().expect("certificates"); + let tls = quic::Identity::open(&certs.cert, &certs.key) + .expect("identity") + .server(&[ALPN], quic::server::ClientAuth::None) + .expect("tls"); + let mut group = c.benchmark_group("worker_request_reply"); + for connections in [1, 16] { + for bytes in [64, 256 * 1024] { + for passes in [1, 4] { + let mut config = Config::default(); + config.passes = NonZeroUsize::new(passes).unwrap(); + let mut worker = Worker::new(config).expect("worker"); + let handle = worker.handle(); + let socket = || { + handle + .udp(UdpSocket::bind("127.0.0.1:0").expect("bind"), udp::Config::default()) + .expect("socket") + }; + let server = quic::Endpoint::new( + socket(), + quic::endpoint::Config::default().with_server(quic::server::Config::new(tls.clone())), + ) + .expect("server"); + let client = quic::Endpoint::new(socket(), quic::endpoint::Config::default()).expect("client"); + let mut dial = quic::client::Config::default(); + dial.alpn = vec![ALPN.to_owned()]; + dial.verify = false; + for _ in 0..connections { + let server = server.clone(); + handle.spawn(async move { + let conn = server.accept().await.expect("accept"); + let (mut send, mut recv) = conn.accept_bi().await.expect("stream"); + let reply = bytes::Bytes::from(vec![0x5a; bytes]); + let mut request = [0; 64]; + while recv.read_exact(&mut request).await.is_ok() { + send.write_all(reply.clone()).await.expect("reply"); + } + }); + } + let (mut streams, _held) = worker + .block_on(async { + let mut streams = Vec::new(); + let mut held = Vec::new(); + for _ in 0..connections { + let conn = client + .connect(&dial, server.local_addr(), "localhost") + .await + .expect("dial"); + streams.push(conn.open_bi(Default::default()).await.expect("stream")); + held.push(conn); + } + (streams, held) + }) + .expect("worker"); + let id = BenchmarkId::new(format!("connections-{connections}/bytes-{bytes}"), passes); + group.throughput(Throughput::Bytes((connections * (64 + bytes)) as u64)); + let mut reply = vec![0; bytes]; + group.bench_function(&id, |b| { + b.iter_custom(|rounds| { + let before = handle.metrics().snapshot(); + let start = Instant::now(); + worker + .block_on(async { + for _ in 0..rounds { + for (send, _) in &mut streams { + send.write_all(&[0xa5; 64][..]).await.expect("request"); + } + for (_, recv) in &mut streams { + recv.read_exact(&mut reply).await.expect("reply"); + assert!(reply.iter().all(|byte| *byte == 0x5a)); + } + } + }) + .expect("worker"); + let elapsed = start.elapsed(); + let after = handle.metrics().snapshot(); + eprintln!( + "{id} rounds={rounds} turns={} passes={} enters={} rx_datagrams={}", + after.turns - before.turns, + after.passes - before.passes, + after.enters - before.enters, + after.rx_datagrams - before.rx_datagrams, + ); + elapsed + }); + }); + } + } + } + group.finish(); +} + +#[cfg(not(all(target_os = "linux", feature = "quic")))] +fn request_reply(_: &mut criterion::Criterion) {} + #[cfg(target_os = "linux")] fn benchmark(c: &mut criterion::Criterion) { use std::net::UdpSocket; @@ -86,5 +201,5 @@ fn local_thread_check(_: &mut criterion::Criterion) {} #[cfg(not(target_os = "linux"))] fn benchmark(_: &mut criterion::Criterion) {} -criterion_group!(benches, benchmark, local_thread_check); +criterion_group!(benches, benchmark, request_reply, local_thread_check); criterion_main!(benches); diff --git a/rs/moq-uring/src/worker.rs b/rs/moq-uring/src/worker.rs index f3a52b8cf56..3a934c09f00 100644 --- a/rs/moq-uring/src/worker.rs +++ b/rs/moq-uring/src/worker.rs @@ -1100,13 +1100,25 @@ mod tests { const ROUNDS: usize = 16; /// For each round trip, the turns from the request's receive to its reply. - fn hand_offs(passes: usize) -> Option> { + fn hand_offs(passes: usize, busy: bool) -> Option> { let mut worker = worker_with(Config { passes: NonZeroUsize::new(passes).unwrap(), ..Config::default() })?; let handle = worker.handle(); let owner = Owner::new(&handle.shared); + let busy_polls = Rc::new(std::cell::Cell::new((0, 0))); + if busy { + let (owner, polls) = (owner.clone(), busy_polls.clone()); + handle.spawn(kio::wait(move |waiter: &kio::Waiter| { + let (turn, count) = polls.get(); + let count = if turn == owner.turn() { count + 1 } else { 1 }; + assert!(count <= passes, "a self-wake ran past the turn's budget"); + polls.set((owner.turn(), count)); + waiter.waker().wake_by_ref(); + Poll::<()>::Pending + })); + } let udp_config = udp::Config { gro: false, gso: false, @@ -1163,13 +1175,20 @@ mod tests { } }) .expect("worker"); + if busy { + assert_eq!(busy_polls.get().1, passes, "the busy task spent every pass"); + } + drop(worker); + assert_eq!(Rc::strong_count(&busy_polls), 1, "a task survived its worker"); Some(gaps.take()) } - let Some(two) = hand_offs(2) else { return }; - assert_eq!(two, [0; ROUNDS], "the reply is staged in its request's turn"); - let Some(one) = hand_offs(1) else { return }; - assert_eq!(one, [1; ROUNDS], "a pass per turn spends a turn on the hand-off"); + for busy in [false, true] { + for passes in [1, 2, 4, 8] { + let Some(gaps) = hand_offs(passes, busy) else { return }; + assert_eq!(gaps, [u64::from(passes == 1); ROUNDS], "passes={passes}, busy={busy}"); + } + } } #[test] From 8b964928d20fc7133a59a663f6d9ee60b77de9ed Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 08:53:27 -0800 Subject: [PATCH 12/13] fix(bench): use a named request-reply case Co-Authored-By: GPT-6.1 Sol --- rs/moq-uring/benches/worker_turns.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rs/moq-uring/benches/worker_turns.rs b/rs/moq-uring/benches/worker_turns.rs index 66e6fbb0e24..bbd65c3a23c 100644 --- a/rs/moq-uring/benches/worker_turns.rs +++ b/rs/moq-uring/benches/worker_turns.rs @@ -17,7 +17,7 @@ mod support; fn request_reply(c: &mut criterion::Criterion) { use std::{net::UdpSocket, num::NonZeroUsize, time::Instant}; - use criterion::{BenchmarkId, Throughput}; + use criterion::Throughput; use moq_uring::{Config, Worker, quic, udp}; const ALPN: &str = "worker-turns"; @@ -76,10 +76,10 @@ fn request_reply(c: &mut criterion::Criterion) { (streams, held) }) .expect("worker"); - let id = BenchmarkId::new(format!("connections-{connections}/bytes-{bytes}"), passes); + let id = format!("connections-{connections}/bytes-{bytes}/passes-{passes}"); group.throughput(Throughput::Bytes((connections * (64 + bytes)) as u64)); let mut reply = vec![0; bytes]; - group.bench_function(&id, |b| { + group.bench_function(id.clone(), |b| { b.iter_custom(|rounds| { let before = handle.metrics().snapshot(); let start = Instant::now(); From 957833090663b9f8af82214907809f0febe050e8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 09:08:38 -0800 Subject: [PATCH 13/13] fix(uring): coalesce held wakes and benchmark task-driven peers Co-Authored-By: GPT-6.1 Sol --- bench/README.md | 2 +- quest/m2/uring-bench-followups.md | 13 +++++---- rs/kio/src/task.rs | 23 +++++++++------ rs/moq-uring/README.md | 8 ++++-- rs/moq-uring/benches/worker_turns.rs | 33 +++++++++++++--------- rs/moq-uring/src/quic/driver/connection.rs | 8 ++---- 6 files changed, 51 insertions(+), 36 deletions(-) diff --git a/bench/README.md b/bench/README.md index 94dac3a0acb..4ca092bd451 100644 --- a/bench/README.md +++ b/bench/README.md @@ -134,7 +134,7 @@ CPUs the run may use. worker, sweeping 1 or 16 connections against 64 B or 256 KiB replies with 1 or 4 task passes. It prints rounds and counter deltas for turns, passes, enters and received datagrams, so counts per round can be compared alongside -timing. The existing nightly `rust-uring` group runs these cases. +timing. The existing nightly `rust-shared` group runs these cases. ```bash nix develop --command cargo bench --locked -p moq-uring --bench worker_turns -- worker_request_reply diff --git a/quest/m2/uring-bench-followups.md b/quest/m2/uring-bench-followups.md index 9b1b3ea2ca2..f0ecf486149 100644 --- a/quest/m2/uring-bench-followups.md +++ b/quest/m2/uring-bench-followups.md @@ -1,17 +1,20 @@ -# [XS] Confirm io_uring quiescence on a quiet host and track its turn counts nightly +# [XS] Confirm io_uring quiescence on a quiet host ## Goal -Run-to-quiescence on the io_uring workers has acceptance numbers from a quiet host (fanout CPU per Gbps and p99 from `just bench-runtime`, against the tokio workers), and a QUIC request-reply variant of the `worker_turns` bench gives its turn and enter counts a nightly series, so a regression in batching shows up. +Run-to-quiescence on the io_uring workers has acceptance numbers from a quiet host (fanout CPU per Gbps and p99 from `just bench-runtime`, against the tokio workers). ## Plan -Follow-ups of [moq-dev/moq#7530](https://github.com/moq-dev/moq/pull/7530), which runs a turn's tasks to quiescence before submitting. Its deterministic counts were the evidence (fanout 40.2 to 20.9 turns per round, -3.6% instructions); its `just bench-runtime` rows were indicative only, because host load put every runtime's p99 in seconds and moved the unchanged tokio rows by up to 8%. This waits on #7530 landing. +Follow-ups of [moq-dev/moq#7530](https://github.com/moq-dev/moq/pull/7530), which runs a turn's tasks to quiescence before submitting. Its deterministic counts were the evidence (fanout 40.2 to 20.9 turns per round, -3.6% instructions); its `just bench-runtime` rows were indicative only, because host load put every runtime's p99 in seconds and moved the unchanged tokio rows by up to 8%. This waits on [moq-dev/moq#7530](https://github.com/moq-dev/moq/pull/7530) landing. Decided 2026-11-01 (maintainer, follow-up planning after the 2026-11-01 /quest-afk run): m2, one quest for both. -- **Quiet re-run.** Record `/proc/loadavg` and run `just bench-runtime` for fanout and fanout-heavy, base against #7530, alternating. [moq-dev/moq#7533](https://github.com/moq-dev/moq/pull/7533) and [moq-dev/moq#7534](https://github.com/moq-dev/moq/pull/7534) saw the load generator's p99 sit near 1.4 s in every run, a startup artifact inside the throughput window, so check whether a quiet host alone fixes p99 or the generator's latency window needs to exclude startup first. Record the result in the [perf epic](/quest/m2/perf/README.md). -- **Nightly series.** #7530's harness (`rs/moq-uring/examples/quiescence/`, in its PR body, uncommitted) drove chat, bulk, and fanout shapes over QUIC and counted instructions, turns, and enters per round. Add a QUIC request-reply variant to the existing `worker_turns` bench (`rs/moq-uring/benches/worker_turns.rs`, already in `sh/gh/nightly.sh`'s bench list), reporting turns and enters per round as that harness did, so the nightly series covers it. +- **Quiet re-run.** Record `/proc/loadavg` and run `just bench-runtime` for fanout and fanout-heavy, base against [moq-dev/moq#7530](https://github.com/moq-dev/moq/pull/7530), alternating. [moq-dev/moq#7533](https://github.com/moq-dev/moq/pull/7533) and [moq-dev/moq#7534](https://github.com/moq-dev/moq/pull/7534) saw the load generator's p99 sit near 1.4 s in every run, a startup artifact inside the throughput window, so check whether a quiet host alone fixes p99 or the generator's latency window needs to exclude startup first. Record the result in the [perf epic](/quest/m2/perf/README.md). +The QUIC request-reply benchmark is now part of `worker_turns` in +[moq-dev/moq#7530](https://github.com/moq-dev/moq/pull/7530). The nightly +`rust-shared` group retains its timing series and turn/enter counter deltas in +the run logs; adding that benchmark is no longer open work here. Public API: none. Wire: none. diff --git a/rs/kio/src/task.rs b/rs/kio/src/task.rs index b6592c15028..373cf6e239b 100644 --- a/rs/kio/src/task.rs +++ b/rs/kio/src/task.rs @@ -210,14 +210,14 @@ impl Polling { POLLING.with(|polling| { polling.open.borrow_mut().pop(); let mut held = polling.held.borrow_mut(); - let mut index = self.start; - while index < held.len() { - match held[index].0 == self.identity { - true => mine.push(held.remove(index).1), - false => index += 1, - } - } + mine.extend( + held.extract_if(self.start.., |(identity, _)| *identity == self.identity) + .map(|(_, shared)| shared), + ); }); + // Keep holding O(1), then coalesce the same set's wakes across passes. + mine.sort_unstable_by_key(Arc::as_ptr); + mine.dedup_by(|a, b| Arc::ptr_eq(a, b)); // Outside the borrow: a waker may poll inline and open a batch of its own. for shared in mine { shared.release(); @@ -927,9 +927,14 @@ mod tests { Poll::Pending }); - let waiter = Waiter::noop(); - assert_eq!(tasks.poll_passes(&waiter, NonZeroUsize::new(4).unwrap()), 4); + let wakes = Arc::new(Wakes(AtomicUsize::new(0))); + let waiter = Waiter::new(Waker::from(wakes.clone())); + let references = Arc::strong_count(&tasks.shared); + let ran = waiter.batch(|| tasks.poll_passes(&waiter, NonZeroUsize::new(4).unwrap())); + assert_eq!(ran, 4); assert_eq!(polls.load(Ordering::SeqCst), 4, "once per pass"); + assert_eq!(wakes.0.load(Ordering::SeqCst), 1, "one wake for the work left over"); + assert_eq!(Arc::strong_count(&tasks.shared), references, "a batch retained a wake"); } /// A retired occupant's leftover waker firing into a reused slot costs at diff --git a/rs/moq-uring/README.md b/rs/moq-uring/README.md index 1f016664d68..fc0905f277a 100644 --- a/rs/moq-uring/README.md +++ b/rs/moq-uring/README.md @@ -121,9 +121,11 @@ skip: in the `uring.yml` release gate and the nightly `rs uring` job. first spike (recv batching x GRO x GSO, epoll vs io\_uring); see git history for their methodology. `echo_quic` is the ablation matrix over the real worker: the same QUIC echo with receive batching, GRO, and GSO toggled one -at a time. `worker_turns` sweeps socket counts and datagrams per burst -independently to measure drive-loop batching without QUIC. Its smoke cases -run in the Rust nightly benchmark group. +at a time. `worker_turns` sweeps sockets against datagrams per burst for UDP, +and connections against reply bytes and task passes for QUIC request/reply. +The QUIC client and server run as worker tasks; the root only waits for a +measurement to finish. Counter deltas stay in the run's log beside the timing +series in the nightly `rust-shared` group. ```bash just rs bench-udp --sample-size 20 --measurement-time 2 --warm-up-time 1 diff --git a/rs/moq-uring/benches/worker_turns.rs b/rs/moq-uring/benches/worker_turns.rs index bbd65c3a23c..43b110ec162 100644 --- a/rs/moq-uring/benches/worker_turns.rs +++ b/rs/moq-uring/benches/worker_turns.rs @@ -79,25 +79,32 @@ fn request_reply(c: &mut criterion::Criterion) { let id = format!("connections-{connections}/bytes-{bytes}/passes-{passes}"); group.throughput(Throughput::Bytes((connections * (64 + bytes)) as u64)); let mut reply = vec![0; bytes]; + let completed = kio::Queue::bounded(1); group.bench_function(id.clone(), |b| { b.iter_custom(|rounds| { + let mut running = std::mem::take(&mut streams); + let mut answer = std::mem::take(&mut reply); + let done = completed.clone(); + handle.spawn(async move { + for _ in 0..rounds { + for (send, _) in &mut running { + send.write_all(&[0xa5; 64][..]).await.expect("request"); + } + for (_, recv) in &mut running { + recv.read_exact(&mut answer).await.expect("reply"); + } + } + done.try_push((running, answer)).expect("complete"); + }); let before = handle.metrics().snapshot(); let start = Instant::now(); - worker - .block_on(async { - for _ in 0..rounds { - for (send, _) in &mut streams { - send.write_all(&[0xa5; 64][..]).await.expect("request"); - } - for (_, recv) in &mut streams { - recv.read_exact(&mut reply).await.expect("reply"); - assert!(reply.iter().all(|byte| *byte == 0x5a)); - } - } - }) - .expect("worker"); + (streams, reply) = worker + .block_on(kio::wait(|waiter| completed.poll_pop(waiter))) + .expect("worker") + .expect("measurement"); let elapsed = start.elapsed(); let after = handle.metrics().snapshot(); + assert!(reply.iter().all(|byte| *byte == 0x5a)); eprintln!( "{id} rounds={rounds} turns={} passes={} enters={} rx_datagrams={}", after.turns - before.turns, diff --git a/rs/moq-uring/src/quic/driver/connection.rs b/rs/moq-uring/src/quic/driver/connection.rs index 671690e89b8..27e981b38b6 100644 --- a/rs/moq-uring/src/quic/driver/connection.rs +++ b/rs/moq-uring/src/quic/driver/connection.rs @@ -959,11 +959,9 @@ impl Driver { self.trains = 0; } while self.trains < TRAINS_PER_TURN { - match self.flush_one(waiter) { - Poll::Ready(Ok(())) => self.trains += 1, - Poll::Ready(Err(err)) => return Poll::Ready(err), - // Backpressure, or nothing left to stage. - Poll::Pending => return Poll::Pending, + match std::task::ready!(self.flush_one(waiter)) { + Ok(()) => self.trains += 1, + Err(err) => return Poll::Ready(err), } // The budget is spent: requeue behind the next reap, once per turn. // If moq-quic is drained by then, that poll costs one empty acquire