diff --git a/rs/kio/benches/tasks.rs b/rs/kio/benches/tasks.rs index ca1bd1e98e0..accda24a341 100644 --- a/rs/kio/benches/tasks.rs +++ b/rs/kio/benches/tasks.rs @@ -13,6 +13,9 @@ //! 16 gates and polls the set once; the woken tasks consume the fire and //! re-park. This is the shape of a driver serving many subscriptions, where //! per-task wakeup granularity is the whole point. +//! `kio_tasks_batch` fires the gates from inside the owner's +//! [`kio::Waiter::batch`], the driver-advancing-its-clock shape, so the +//! difference is what holding and dropping the set's wake of the owner costs. //! - `spawn_drain_1000`: cold churn. A fresh set per iteration pushes 1000 //! immediately-ready tasks and drains them, measuring per-task setup and //! retirement (allocation included). @@ -116,6 +119,29 @@ fn wake_sparse(c: &mut Criterion) { }); }); + // The driver shape: the owner fires its own tasks' gates mid-poll, the way + // advancing its clock does, inside the batch that holds the set's wake of it. + group.bench_function("kio_tasks_batch", |b| { + let gates: Vec<_> = (0..N).map(|_| Gate::new()).collect(); + let mut tasks = kio::Tasks::new(); + for gate in &gates { + tasks.push(forever(gate.clone())); + } + let waiter = kio::Waiter::noop(); + let _ = tasks.poll(&waiter); + + let mut next = 0; + b.iter(|| { + waiter.batch(|| { + for _ in 0..K { + gates[next].fire(); + next = (next + 1) % N; + } + let _ = black_box(tasks.poll(&waiter)); + }); + }); + }); + group.bench_function("futures_unordered", |b| { let gates: Vec<_> = (0..N).map(|_| Gate::new()).collect(); let mut set = futures::stream::FuturesUnordered::new(); diff --git a/rs/kio/src/loom.rs b/rs/kio/src/loom.rs index 20f66429511..ae0e1052dbc 100644 --- a/rs/kio/src/loom.rs +++ b/rs/kio/src/loom.rs @@ -334,3 +334,44 @@ fn consumer_creation_cannot_cross_an_idle_guard() { drop(minting.join().unwrap()); }); } + +/// Ready once the gate opens. +fn gated(gate: crate::Consumer) -> impl FnMut(&crate::Waiter) -> Poll<()> + Unpin { + move |waiter| match gate.poll(waiter, |open| if **open { Poll::Ready(()) } else { Poll::Pending }) { + Poll::Ready(_) => Poll::Ready(()), + Poll::Pending => Poll::Pending, + } +} + +/// The owner's poll wakes one of its own tasks, holding that wake of itself until it returns, +/// while another thread wakes the other task. Neither wake may be lost, whether the +/// owner's poll takes its own wake in the same pass or returns first. +#[test] +fn a_held_wake_never_hides_a_remote_wake() { + for same_pass in [false, true] { + loom::model(move || { + let local = Producer::new(false); + let remote = Producer::new(false); + let mut tasks = crate::Tasks::new(); + tasks.push(gated(local.consume())); + tasks.push(gated(remote.consume())); + + let opener = thread::spawn(move || *remote.write().ok().expect("open") = true); + + let mut first = true; + block_on(wait(move |waiter| { + waiter.batch(|| { + if std::mem::take(&mut first) { + let _ = tasks.poll(waiter); + *local.write().ok().expect("open") = true; + if !same_pass { + return Poll::Pending; + } + } + tasks.poll(waiter) + }) + })); + opener.join().unwrap(); + }); + } +} diff --git a/rs/kio/src/sync.rs b/rs/kio/src/sync.rs index 7b70b9cad7a..e864676f121 100644 --- a/rs/kio/src/sync.rs +++ b/rs/kio/src/sync.rs @@ -11,6 +11,12 @@ //! `Arc` that carries the producer/consumer counts is loom's. #[cfg(loom)] -pub(crate) use loom::sync::{Arc, Mutex, MutexGuard, atomic::AtomicUsize}; +pub(crate) use loom::sync::{ + Arc, Mutex, MutexGuard, + atomic::{AtomicBool, AtomicU64, AtomicUsize}, +}; #[cfg(not(loom))] -pub(crate) use std::sync::{Arc, Mutex, MutexGuard, atomic::AtomicUsize}; +pub(crate) use std::sync::{ + Arc, Mutex, MutexGuard, + atomic::{AtomicBool, AtomicU64, AtomicUsize}, +}; diff --git a/rs/kio/src/task.rs b/rs/kio/src/task.rs index 0637818bfe4..5828961b0cc 100644 --- a/rs/kio/src/task.rs +++ b/rs/kio/src/task.rs @@ -34,18 +34,22 @@ //! pinned futures, a slot's identity is stable and its [`Waker`] is minted once //! and reused across occupants: the per-task allocation `FuturesUnordered` //! pays per future (and futures-buffered removes with an unsafe arena) is a -//! per-slot cost here, amortized to zero in a long-lived set. Safe Rust +//! per-slot cost here, amortized to zero in a long-lived set. An owner that +//! wakes its own tasks mid-poll (a driver advancing its clock) runs that poll in +//! [`Waiter::batch`], so the set's wake of the owner is not a second poll. Safe Rust //! throughout. use std::{ - sync::{ - Arc, Mutex, - atomic::{AtomicBool, AtomicU64, Ordering}, - }, - task::{Context, Poll, Wake, Waker}, + cell::{Cell, RefCell}, + // std's `Arc`: `Waker::from` needs it. The atomics are loom's under `--cfg loom`. + sync::{Arc, atomic::Ordering}, + task::{Context, Poll, RawWakerVTable, Wake, Waker}, }; -use crate::waiter::{Park, Waiter}; +use crate::{ + sync::{AtomicBool, AtomicU64, Mutex}, + waiter::{Park, Waiter}, +}; /// Wake-bits per chunk word and words per chunk: one chunk covers 1024 slots. const WORD_BITS: usize = 64; @@ -106,17 +110,110 @@ struct Shared { } impl Shared { - fn wake_parent(&self) { + fn wake_parent(self: &Arc) { if self.pending.swap(true, Ordering::AcqRel) { return; } let parent = self.parent.lock().unwrap().clone(); + if let Some(waker) = parent + && !Polling::hold(&waker, self) + { + waker.wake(); + } + } + + /// Deliver a wake [`Polling::hold`] kept back, unless the set was polled since. + fn release(&self) { + if !self.pending.load(Ordering::Acquire) { + return; + } + let parent = self.parent.lock().unwrap().clone(); if let Some(waker) = parent { waker.wake(); } } } +// Loom runs its threads on one OS thread, so the scope must be loom's to stay per thread. +#[cfg(not(loom))] +std::thread_local! { + static POLLING: Polling = const { Polling { current: Cell::new(None), held: RefCell::new(Vec::new()) } }; +} +#[cfg(loom)] +loom::thread_local! { + static POLLING: Polling = Polling { current: Cell::new(None), held: RefCell::new(Vec::new()) }; +} + +/// A waker's identity: what [`Waker::will_wake`] compares. +type Identity = (*const (), *const RawWakerVTable); + +/// The [`Waiter::batch`] running on this thread, and the wakes held for it. +/// +/// A set woken from inside its owner's poll (by time that poll advanced, say) +/// would wake a task that is already running, and the executor would poll it +/// again for nothing once the set's own poll, later in the same pass, took the +/// wake. So the wake is held until the batch returns, then delivered only if the +/// set is still pending. A wake from another thread is never held: it cannot +/// know where the owner's poll is. +pub(crate) struct Polling { + /// The innermost batch's waker; the outer ones wait on their stack frames. + current: Cell>, + /// Held wakes. Each batch owns the ones past the length it found on entry. + held: RefCell>>, +} + +impl Polling { + /// Run `poll` as the task `waker` wakes, delivering the wakes it held when it returns. + pub(crate) fn run(waker: &Waker, poll: impl FnOnce() -> R) -> R { + struct Scope { + outer: Option, + start: usize, + } + impl Drop for Scope { + fn drop(&mut self) { + let held: Vec<_> = POLLING.with(|polling| { + polling.current.set(self.outer); + let mut held = polling.held.borrow_mut(); + match held.len() > self.start { + true => held.drain(self.start..).collect(), + false => Vec::new(), + } + }); + // Outside the borrow: a waker may poll inline and open a batch of its own. + for shared in held { + shared.release(); + } + } + } + + // Delivered on unwind too, so a panicking poll loses no wakeup. Polled from a + // thread-local's destructor after the scope is gone, nothing is held at all. + let _scope = POLLING + .try_with(|polling| Scope { + outer: polling.current.replace(Some((waker.data(), waker.vtable()))), + start: polling.held.borrow().len(), + }) + .ok(); + poll() + } + + /// Keep back a wake of `parent` when it is the innermost batch on this thread. + /// + /// A wake from a thread-local's destructor may run after the scope is gone; it + /// is delivered. + fn hold(parent: &Waker, shared: &Arc) -> bool { + POLLING + .try_with(|polling| { + if polling.current.get() != Some((parent.data(), parent.vtable())) { + return false; + } + polling.held.borrow_mut().push(shared.clone()); + true + }) + .unwrap_or(false) + } +} + /// The per-slot waker: sets the slot's wake bit and nudges the owner. struct SlotWaker { chunk: Arc, @@ -546,6 +643,115 @@ mod tests { assert_eq!(b_polls.load(Ordering::SeqCst), 1, "one poll, not two"); } + /// Counts the owner's wakes. + struct Wakes(AtomicUsize); + + impl Wake for Wakes { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + /// A task woken by a thread-local's destructor, after the batch scope's own + /// thread-local is gone, still wakes the owner rather than panicking there. + #[test] + fn a_wake_during_thread_teardown_reaches_the_owner() { + type Set = Tasks Poll<()>>>; + struct Teardown { + gate: crate::Producer, + _tasks: Set, + } + impl Drop for Teardown { + fn drop(&mut self) { + open(&self.gate); + } + } + std::thread_local! { + static TEARDOWN: RefCell> = const { RefCell::new(None) }; + } + + let wakes = Arc::new(Wakes(AtomicUsize::new(0))); + let waker = Waker::from(wakes.clone()); + std::thread::spawn(move || { + // Touched before the batch scope, so destroyed after it: destructors run LIFO. + TEARDOWN.with(|_| {}); + let gate = crate::Producer::new(false); + let mut tasks: Set = Tasks::new(); + tasks.push(Box::new(gated(gate.consume(), Arc::new(AtomicUsize::new(0))))); + let waiter = Waiter::new(waker); + assert!(waiter.batch(|| tasks.poll(&waiter)).is_pending()); + TEARDOWN.with(|teardown| *teardown.borrow_mut() = Some(Teardown { gate, _tasks: tasks })); + }) + .join() + .unwrap(); + assert_eq!(wakes.0.load(Ordering::SeqCst), 1, "the teardown wake was lost"); + } + + /// An owner whose poll wakes a task in its own set, then polls the set, took that + /// wake in the same pass: the executor must not be asked to poll it again. + #[test] + fn a_wake_from_the_owners_own_poll_is_not_redelivered() { + let mut tasks = Tasks::new(); + let gate = crate::Producer::new(false); + let polls = Arc::new(AtomicUsize::new(0)); + tasks.push(gated(gate.consume(), polls.clone())); + let _parked = crate::Producer::new(false); + tasks.push(gated(_parked.consume(), Arc::new(AtomicUsize::new(0)))); + + let opening = std::cell::Cell::new(false); + let mut owner = std::pin::pin!(crate::wait(|waiter: &Waiter| { + waiter.batch(|| { + if opening.take() { + open(&gate); + } + tasks.poll(waiter) + }) + })); + let wakes = Arc::new(Wakes(AtomicUsize::new(0))); + let waker = Waker::from(wakes.clone()); + let mut cx = Context::from_waker(&waker); + + assert!(owner.as_mut().poll(&mut cx).is_pending()); + opening.set(true); + assert!(owner.as_mut().poll(&mut cx).is_pending()); + assert_eq!(polls.load(Ordering::SeqCst), 2, "the woken task ran in that pass"); + assert_eq!( + wakes.0.load(Ordering::SeqCst), + 0, + "the owner was woken for a wake it took" + ); + } + + /// The same wake, when the owner's poll returns without polling the set again, + /// still reaches the owner as it returns. + #[test] + fn a_held_wake_lands_when_the_set_is_not_polled() { + let mut tasks = Tasks::new(); + let gate = crate::Producer::new(false); + let polls = Arc::new(AtomicUsize::new(0)); + tasks.push(gated(gate.consume(), polls.clone())); + + let mut first = true; + let mut owner = std::pin::pin!(crate::wait(|waiter: &Waiter| { + waiter.batch(|| { + if std::mem::take(&mut first) { + let _ = tasks.poll(waiter); + open(&gate); + return Poll::Pending; + } + tasks.poll(waiter) + }) + })); + let wakes = Arc::new(Wakes(AtomicUsize::new(0))); + let waker = Waker::from(wakes.clone()); + let mut cx = Context::from_waker(&waker); + + assert!(owner.as_mut().poll(&mut cx).is_pending()); + assert_eq!(wakes.0.load(Ordering::SeqCst), 1, "the held wake was lost"); + assert!(owner.as_mut().poll(&mut cx).is_ready()); + assert_eq!(polls.load(Ordering::SeqCst), 2); + } + /// A task is allowed to self-wake before returning Pending. Such a task must /// be polled once per owner poll, not spun on inside it, or it starves the /// caller's other arms; the re-queue instead wakes the owner for the next diff --git a/rs/kio/src/waiter.rs b/rs/kio/src/waiter.rs index 3ec164af6f1..753a9c8c31c 100644 --- a/rs/kio/src/waiter.rs +++ b/rs/kio/src/waiter.rs @@ -140,6 +140,20 @@ impl Waiter { true } + /// Run one poll of this waiter's task, the way a driver's `poll` does. + /// + /// A [`Tasks`](crate::Tasks) set this waiter polls, woken from inside `poll` + /// (by time it advances, or a channel an earlier arm writes), holds its wake of + /// the task until `poll` returns, and drops it if `poll` polled the set since. + /// Otherwise the executor polls the task again for a wake its own poll took. + /// + /// Use it once, around a task's outermost poll. Only the innermost batch holds + /// wakes, so one nested inside (around a [`Tasks`](crate::Tasks) child, say) + /// lets its siblings' wakes of the owner through, and costs polls. + pub fn batch(&self, poll: impl FnOnce() -> R) -> R { + crate::task::Polling::run(&self.waker, poll) + } + /// Poll a foreign [`Future`] against this waiter, so it re-wakes the enclosing /// `poll_*` step when it is ready. pub fn poll_future(&self, future: Pin<&mut F>) -> Poll { diff --git a/rs/moq-net/src/driver.rs b/rs/moq-net/src/driver.rs index 5e7057b266d..8cb0d7dd41a 100644 --- a/rs/moq-net/src/driver.rs +++ b/rs/moq-net/src/driver.rs @@ -61,13 +61,14 @@ impl Driver { /// /// Panics if `now` is earlier than the previous poll or construction time. pub fn poll(&mut self, now: Instant, waiter: &kio::Waiter) -> Result, Error> { - self.clock.advance(now); - self.clock.register_driver(waiter); - match self.state.poll(waiter) { - Poll::Ready(Ok(())) => Err(Error::Closed), - Poll::Ready(Err(err)) => Err(err), - Poll::Pending => Ok(self.clock.timeout()), - } + waiter.batch(|| { + self.clock.advance(now); + match self.state.poll(waiter) { + Poll::Ready(Ok(())) => Err(Error::Closed), + Poll::Ready(Err(err)) => Err(err), + Poll::Pending => Ok(self.clock.park(waiter)), + } + }) } } diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 2cd63720271..a43273c52c7 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2278,14 +2278,15 @@ impl Driver { /// broadcast, or [`Dynamic`]) has dropped and the remaining lifecycle work /// has drained. pub fn poll(&mut self, now: Instant, waiter: &kio::Waiter) -> Result, Error> { - self.timers.advance(now); - self.timers.register_driver(waiter); - let result = self.state.poll(waiter); - let gc = self.pool.gc(now); - if result.is_ready() { - return Err(Error::Closed); - } - Ok(self.timers.timeout().into_iter().chain(gc).min()) + waiter.batch(|| { + self.timers.advance(now); + let result = self.state.poll(waiter); + let gc = self.pool.gc(now); + if result.is_ready() { + return Err(Error::Closed); + } + Ok(self.timers.park(waiter).into_iter().chain(gc).min()) + }) } } diff --git a/rs/moq-net/src/time.rs b/rs/moq-net/src/time.rs index 839199e251e..459bec8d4f1 100644 --- a/rs/moq-net/src/time.rs +++ b/rs/moq-net/src/time.rs @@ -73,7 +73,7 @@ pub(crate) async fn run_sim(mut driver: D) -> Error { /// A clock private to one driver, advanced only by the instants it is polled with. /// /// Clones share the clock. Everything the driver owns arms its [`Deadline`]s here, -/// and [`Self::timeout`] reports the earliest so the driver can ask to be polled +/// and [`Self::park`] reports the earliest so the driver can ask to be polled /// again by then. #[derive(Clone, Default)] pub(crate) struct Clock(Arc>); @@ -112,11 +112,17 @@ impl Clock { } /// Move to `now`, waking every deadline at or before it. + /// + /// The driver is polling from here until [`Self::park`], which reads every + /// deadline armed meanwhile, so it drops its registration rather than wake + /// itself. Taken rather than replaced: a new list's serial would cost the + /// driver's waiter its records of every other list it is parked on. pub(crate) fn advance(&self, now: Instant) { let due = { let mut state = self.0.lock().unwrap(); assert!(state.now.is_none_or(|prev| now >= prev), "driver time moved backwards"); state.now = Some(now); + drop(state.driver.take()); let mut due = Vec::new(); while state.deadlines.first_key_value().is_some_and(|((at, _), _)| *at <= now) { due.push(state.deadlines.pop_first().unwrap().1); @@ -135,12 +141,17 @@ impl Clock { self.0.lock().unwrap().now } - /// Register the driver's poll to be woken by a new earliest deadline. - pub(crate) fn register_driver(&self, waiter: &kio::Waiter) { - waiter.register(&mut self.0.lock().unwrap().driver); + /// The earliest armed deadline, registering the driver's poll to be woken when an + /// earlier one is armed. Ends the poll [`Self::advance`] began: one lock, so a + /// deadline armed elsewhere is either reported here or wakes the driver. + pub(crate) fn park(&self, waiter: &kio::Waiter) -> Option { + let mut state = self.0.lock().unwrap(); + waiter.register(&mut state.driver); + state.deadlines.first_key_value().map(|((at, _), _)| *at) } /// The earliest armed deadline still in the future. + #[cfg(test)] pub(crate) fn timeout(&self) -> Option { self.0 .lock() @@ -320,6 +331,68 @@ mod tests { assert!(deadline.poll(&waiter).is_ready()); } + /// A deadline that fires and re-arms inside the driver's poll is that poll's work: + /// neither its wake nor its re-arm asks for another. One armed from outside the + /// poll still wakes the driver. + #[test] + fn a_deadline_costs_the_driver_one_poll() { + struct Wakes(std::sync::atomic::AtomicUsize); + impl std::task::Wake for Wakes { + fn wake(self: Arc) { + self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + } + + const PERIOD: Duration = Duration::from_secs(1); + let start = Instant::now(); + let clock = Clock::new(start); + + // A driver child that re-arms its deadline every time it fires, like PROBE's refresh. + let mut tasks = kio::Tasks::new(); + let mut tick = Deadline::after(&clock, PERIOD); + let ticking = clock.clone(); + tasks.push(move |waiter: &kio::Waiter| { + while tick.poll(waiter).is_ready() { + tick.set(Some(ticking.now() + PERIOD)); + } + Poll::Pending + }); + + // `Driver::poll`, minus the protocol. + let now = std::cell::Cell::new(start); + let timeout = std::cell::Cell::new(None); + let mut driver = std::pin::pin!(kio::wait(|waiter: &kio::Waiter| { + waiter.batch(|| { + clock.advance(now.get()); + let _ = tasks.poll(waiter); + timeout.set(clock.park(waiter)); + }); + Poll::<()>::Pending + })); + let wakes = Arc::new(Wakes(std::sync::atomic::AtomicUsize::new(0))); + let waker = std::task::Waker::from(wakes.clone()); + let mut cx = std::task::Context::from_waker(&waker); + + assert!(driver.as_mut().poll(&mut cx).is_pending()); + for ticks in 1..=3 { + now.set(start + PERIOD * ticks); + assert!(driver.as_mut().poll(&mut cx).is_pending()); + assert_eq!(timeout.get(), Some(now.get() + PERIOD), "the deadline re-armed"); + assert_eq!( + wakes.0.load(std::sync::atomic::Ordering::SeqCst), + 0, + "tick {ticks} woke the driver it ran in" + ); + } + + let _sooner = Deadline::after(&clock, PERIOD / 2); + assert_eq!( + wakes.0.load(std::sync::atomic::Ordering::SeqCst), + 1, + "an earlier deadline armed outside the poll must wake the driver" + ); + } + #[test] #[should_panic(expected = "driver time moved backwards")] fn time_cannot_move_backwards() { diff --git a/rs/moq-net/tests/idle_session.rs b/rs/moq-net/tests/idle_session.rs index 61ce06e0d1c..26081742c90 100644 --- a/rs/moq-net/tests/idle_session.rs +++ b/rs/moq-net/tests/idle_session.rs @@ -68,11 +68,11 @@ async fn idle_viewer_does_not_wake_the_server() { let polls = polls.load(Ordering::Relaxed) - start_polls; let reads = reads.stats_reads() - start_reads; - // Nothing moves, so the only work left is the max-age refresh: one sample, - // and the deadline's poll plus the re-poll its own wake schedules. + // Nothing moves, so the only work left is the max-age refresh: one sample and + // one driver poll each. The deadline fires and re-arms inside that poll. let refreshes = (IDLE.as_secs() / PROBE_MAX_AGE.as_secs()) as usize; assert!(reads <= refreshes, "{version}: {reads} stats reads over {IDLE:?}"); - assert!(polls <= 2 * refreshes, "{version}: {polls} driver polls over {IDLE:?}"); + assert!(polls <= refreshes, "{version}: {polls} driver polls over {IDLE:?}"); } }