Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions rs/kio/benches/tasks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
//! 16 gates and polls the set once; the woken tasks consume the fire and
//! re-park. This is the shape of a driver serving many subscriptions, where
//! per-task wakeup granularity is the whole point.
//! `kio_tasks_batch` fires the gates from inside the owner's
//! [`kio::Waiter::batch`], the driver-advancing-its-clock shape, so the
//! difference is what holding and dropping the set's wake of the owner costs.
//! - `spawn_drain_1000`: cold churn. A fresh set per iteration pushes 1000
//! immediately-ready tasks and drains them, measuring per-task setup and
//! retirement (allocation included).
Expand Down Expand Up @@ -116,6 +119,29 @@ fn wake_sparse(c: &mut Criterion) {
});
});

// The driver shape: the owner fires its own tasks' gates mid-poll, the way
// advancing its clock does, inside the batch that holds the set's wake of it.
group.bench_function("kio_tasks_batch", |b| {
let gates: Vec<_> = (0..N).map(|_| Gate::new()).collect();
let mut tasks = kio::Tasks::new();
for gate in &gates {
tasks.push(forever(gate.clone()));
}
let waiter = kio::Waiter::noop();
let _ = tasks.poll(&waiter);

let mut next = 0;
b.iter(|| {
waiter.batch(|| {
for _ in 0..K {
gates[next].fire();
next = (next + 1) % N;
}
let _ = black_box(tasks.poll(&waiter));
});
});
});

group.bench_function("futures_unordered", |b| {
let gates: Vec<_> = (0..N).map(|_| Gate::new()).collect();
let mut set = futures::stream::FuturesUnordered::new();
Expand Down
41 changes: 41 additions & 0 deletions rs/kio/src/loom.rs
Original file line number Diff line number Diff line change
Expand Up @@ -334,3 +334,44 @@ fn consumer_creation_cannot_cross_an_idle_guard() {
drop(minting.join().unwrap());
});
}

/// Ready once the gate opens.
fn gated(gate: crate::Consumer<bool>) -> impl FnMut(&crate::Waiter) -> Poll<()> + Unpin {
move |waiter| match gate.poll(waiter, |open| if **open { Poll::Ready(()) } else { Poll::Pending }) {
Poll::Ready(_) => Poll::Ready(()),
Poll::Pending => Poll::Pending,
}
}

/// The owner's poll wakes one of its own tasks, holding that wake of itself until it returns,
/// while another thread wakes the other task. Neither wake may be lost, whether the
/// owner's poll takes its own wake in the same pass or returns first.
#[test]
fn a_held_wake_never_hides_a_remote_wake() {
for same_pass in [false, true] {
loom::model(move || {
let local = Producer::new(false);
let remote = Producer::new(false);
let mut tasks = crate::Tasks::new();
tasks.push(gated(local.consume()));
tasks.push(gated(remote.consume()));

let opener = thread::spawn(move || *remote.write().ok().expect("open") = true);

let mut first = true;
block_on(wait(move |waiter| {
waiter.batch(|| {
if std::mem::take(&mut first) {
let _ = tasks.poll(waiter);
*local.write().ok().expect("open") = true;
if !same_pass {
return Poll::Pending;
}
}
tasks.poll(waiter)
})
}));
opener.join().unwrap();
});
}
}
10 changes: 8 additions & 2 deletions rs/kio/src/sync.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,12 @@
//! `Arc<Counts>` that carries the producer/consumer counts is loom's.

#[cfg(loom)]
pub(crate) use loom::sync::{Arc, Mutex, MutexGuard, atomic::AtomicUsize};
pub(crate) use loom::sync::{
Arc, Mutex, MutexGuard,
atomic::{AtomicBool, AtomicU64, AtomicUsize},
};
#[cfg(not(loom))]
pub(crate) use std::sync::{Arc, Mutex, MutexGuard, atomic::AtomicUsize};
pub(crate) use std::sync::{
Arc, Mutex, MutexGuard,
atomic::{AtomicBool, AtomicU64, AtomicUsize},
};
222 changes: 214 additions & 8 deletions rs/kio/src/task.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,18 +34,22 @@
//! pinned futures, a slot's identity is stable and its [`Waker`] is minted once
//! and reused across occupants: the per-task allocation `FuturesUnordered`
//! pays per future (and futures-buffered removes with an unsafe arena) is a
//! per-slot cost here, amortized to zero in a long-lived set. Safe Rust
//! per-slot cost here, amortized to zero in a long-lived set. An owner that
//! wakes its own tasks mid-poll (a driver advancing its clock) runs that poll in
//! [`Waiter::batch`], so the set's wake of the owner is not a second poll. Safe Rust
//! throughout.

use std::{
sync::{
Arc, Mutex,
atomic::{AtomicBool, AtomicU64, Ordering},
},
task::{Context, Poll, Wake, Waker},
cell::{Cell, RefCell},
// std's `Arc`: `Waker::from` needs it. The atomics are loom's under `--cfg loom`.
sync::{Arc, atomic::Ordering},
task::{Context, Poll, RawWakerVTable, Wake, Waker},
};

use crate::waiter::{Park, Waiter};
use crate::{
sync::{AtomicBool, AtomicU64, Mutex},
waiter::{Park, Waiter},
};

/// Wake-bits per chunk word and words per chunk: one chunk covers 1024 slots.
const WORD_BITS: usize = 64;
Expand Down Expand Up @@ -106,17 +110,110 @@ struct Shared {
}

impl Shared {
fn wake_parent(&self) {
fn wake_parent(self: &Arc<Self>) {
if self.pending.swap(true, Ordering::AcqRel) {
return;
}
let parent = self.parent.lock().unwrap().clone();
if let Some(waker) = parent
&& !Polling::hold(&waker, self)
{
waker.wake();
}
}

/// Deliver a wake [`Polling::hold`] kept back, unless the set was polled since.
fn release(&self) {
if !self.pending.load(Ordering::Acquire) {
return;
}
let parent = self.parent.lock().unwrap().clone();
if let Some(waker) = parent {
waker.wake();
}
}
}

// Loom runs its threads on one OS thread, so the scope must be loom's to stay per thread.
#[cfg(not(loom))]
std::thread_local! {
static POLLING: Polling = const { Polling { current: Cell::new(None), held: RefCell::new(Vec::new()) } };
}
#[cfg(loom)]
loom::thread_local! {
static POLLING: Polling = Polling { current: Cell::new(None), held: RefCell::new(Vec::new()) };
}

/// A waker's identity: what [`Waker::will_wake`] compares.
type Identity = (*const (), *const RawWakerVTable);

/// The [`Waiter::batch`] running on this thread, and the wakes held for it.
///
/// A set woken from inside its owner's poll (by time that poll advanced, say)
/// would wake a task that is already running, and the executor would poll it
/// again for nothing once the set's own poll, later in the same pass, took the
/// wake. So the wake is held until the batch returns, then delivered only if the
/// set is still pending. A wake from another thread is never held: it cannot
/// know where the owner's poll is.
pub(crate) struct Polling {
/// The innermost batch's waker; the outer ones wait on their stack frames.
current: Cell<Option<Identity>>,
/// Held wakes. Each batch owns the ones past the length it found on entry.
held: RefCell<Vec<Arc<Shared>>>,
}

impl Polling {
/// Run `poll` as the task `waker` wakes, delivering the wakes it held when it returns.
pub(crate) fn run<R>(waker: &Waker, poll: impl FnOnce() -> R) -> R {
struct Scope {
outer: Option<Identity>,
start: usize,
}
impl Drop for Scope {
fn drop(&mut self) {
let held: Vec<_> = POLLING.with(|polling| {
polling.current.set(self.outer);
let mut held = polling.held.borrow_mut();
match held.len() > self.start {
true => held.drain(self.start..).collect(),
false => Vec::new(),
}
});
// Outside the borrow: a waker may poll inline and open a batch of its own.
for shared in held {
shared.release();
}
}
}

// Delivered on unwind too, so a panicking poll loses no wakeup. Polled from a
// thread-local's destructor after the scope is gone, nothing is held at all.
let _scope = POLLING
.try_with(|polling| Scope {
outer: polling.current.replace(Some((waker.data(), waker.vtable()))),
start: polling.held.borrow().len(),
})
.ok();
poll()
}

/// Keep back a wake of `parent` when it is the innermost batch on this thread.
///
/// A wake from a thread-local's destructor may run after the scope is gone; it
/// is delivered.
fn hold(parent: &Waker, shared: &Arc<Shared>) -> bool {
POLLING
.try_with(|polling| {
if polling.current.get() != Some((parent.data(), parent.vtable())) {
return false;
}
polling.held.borrow_mut().push(shared.clone());
true
})
.unwrap_or(false)
}
}

/// The per-slot waker: sets the slot's wake bit and nudges the owner.
struct SlotWaker {
chunk: Arc<Chunk>,
Expand Down Expand Up @@ -546,6 +643,115 @@ mod tests {
assert_eq!(b_polls.load(Ordering::SeqCst), 1, "one poll, not two");
}

/// Counts the owner's wakes.
struct Wakes(AtomicUsize);

impl Wake for Wakes {
fn wake(self: Arc<Self>) {
self.0.fetch_add(1, Ordering::SeqCst);
}
}

/// A task woken by a thread-local's destructor, after the batch scope's own
/// thread-local is gone, still wakes the owner rather than panicking there.
#[test]
fn a_wake_during_thread_teardown_reaches_the_owner() {
type Set = Tasks<Box<dyn FnMut(&Waiter) -> Poll<()>>>;
struct Teardown {
gate: crate::Producer<bool>,
_tasks: Set,
}
impl Drop for Teardown {
fn drop(&mut self) {
open(&self.gate);
}
}
std::thread_local! {
static TEARDOWN: RefCell<Option<Teardown>> = const { RefCell::new(None) };
}

let wakes = Arc::new(Wakes(AtomicUsize::new(0)));
let waker = Waker::from(wakes.clone());
std::thread::spawn(move || {
// Touched before the batch scope, so destroyed after it: destructors run LIFO.
TEARDOWN.with(|_| {});
let gate = crate::Producer::new(false);
let mut tasks: Set = Tasks::new();
tasks.push(Box::new(gated(gate.consume(), Arc::new(AtomicUsize::new(0)))));
let waiter = Waiter::new(waker);
assert!(waiter.batch(|| tasks.poll(&waiter)).is_pending());
TEARDOWN.with(|teardown| *teardown.borrow_mut() = Some(Teardown { gate, _tasks: tasks }));
})
.join()
.unwrap();
assert_eq!(wakes.0.load(Ordering::SeqCst), 1, "the teardown wake was lost");
}

/// An owner whose poll wakes a task in its own set, then polls the set, took that
/// wake in the same pass: the executor must not be asked to poll it again.
#[test]
fn a_wake_from_the_owners_own_poll_is_not_redelivered() {
let mut tasks = Tasks::new();
let gate = crate::Producer::new(false);
let polls = Arc::new(AtomicUsize::new(0));
tasks.push(gated(gate.consume(), polls.clone()));
let _parked = crate::Producer::new(false);
tasks.push(gated(_parked.consume(), Arc::new(AtomicUsize::new(0))));

let opening = std::cell::Cell::new(false);
let mut owner = std::pin::pin!(crate::wait(|waiter: &Waiter| {
waiter.batch(|| {
if opening.take() {
open(&gate);
}
tasks.poll(waiter)
})
}));
let wakes = Arc::new(Wakes(AtomicUsize::new(0)));
let waker = Waker::from(wakes.clone());
let mut cx = Context::from_waker(&waker);

assert!(owner.as_mut().poll(&mut cx).is_pending());
opening.set(true);
assert!(owner.as_mut().poll(&mut cx).is_pending());
assert_eq!(polls.load(Ordering::SeqCst), 2, "the woken task ran in that pass");
assert_eq!(
wakes.0.load(Ordering::SeqCst),
0,
"the owner was woken for a wake it took"
);
}

/// The same wake, when the owner's poll returns without polling the set again,
/// still reaches the owner as it returns.
#[test]
fn a_held_wake_lands_when_the_set_is_not_polled() {
let mut tasks = Tasks::new();
let gate = crate::Producer::new(false);
let polls = Arc::new(AtomicUsize::new(0));
tasks.push(gated(gate.consume(), polls.clone()));

let mut first = true;
let mut owner = std::pin::pin!(crate::wait(|waiter: &Waiter| {
waiter.batch(|| {
if std::mem::take(&mut first) {
let _ = tasks.poll(waiter);
open(&gate);
return Poll::Pending;
}
tasks.poll(waiter)
})
}));
let wakes = Arc::new(Wakes(AtomicUsize::new(0)));
let waker = Waker::from(wakes.clone());
let mut cx = Context::from_waker(&waker);

assert!(owner.as_mut().poll(&mut cx).is_pending());
assert_eq!(wakes.0.load(Ordering::SeqCst), 1, "the held wake was lost");
assert!(owner.as_mut().poll(&mut cx).is_ready());
assert_eq!(polls.load(Ordering::SeqCst), 2);
}

/// A task is allowed to self-wake before returning Pending. Such a task must
/// be polled once per owner poll, not spun on inside it, or it starves the
/// caller's other arms; the re-queue instead wakes the owner for the next
Expand Down
14 changes: 14 additions & 0 deletions rs/kio/src/waiter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,20 @@ impl Waiter {
true
}

/// Run one poll of this waiter's task, the way a driver's `poll` does.
///
/// A [`Tasks`](crate::Tasks) set this waiter polls, woken from inside `poll`
/// (by time it advances, or a channel an earlier arm writes), holds its wake of
/// the task until `poll` returns, and drops it if `poll` polled the set since.
/// Otherwise the executor polls the task again for a wake its own poll took.
///
/// Use it once, around a task's outermost poll. Only the innermost batch holds
/// wakes, so one nested inside (around a [`Tasks`](crate::Tasks) child, say)
/// lets its siblings' wakes of the owner through, and costs polls.
pub fn batch<R>(&self, poll: impl FnOnce() -> R) -> R {
crate::task::Polling::run(&self.waker, poll)
}

/// Poll a foreign [`Future`] against this waiter, so it re-wakes the enclosing
/// `poll_*` step when it is ready.
pub fn poll_future<F: Future + ?Sized>(&self, future: Pin<&mut F>) -> Poll<F::Output> {
Expand Down
Loading
Loading