diff --git a/quest/m2/perf/README.md b/quest/m2/perf/README.md index 8d8a6649f9c..10bcf528877 100644 --- a/quest/m2/perf/README.md +++ b/quest/m2/perf/README.md @@ -88,7 +88,6 @@ revisions on the same host and worker count, and profiles with - [Ingest batch](/quest/m2/perf/ingest-batch.md) - relay ingest pays one lock, wake, and clock read per burst of whole frames instead of per frame - [kio waiter costs](/quest/m2/perf/kio-waiter-costs.md) - long-lived waiters stop being rebuilt every poll, IETF fetch keeps one close watch per stream, and a short `kio::log` costs less - [Shard by default](/quest/m2/perf/shard-default.md) - a saturating shard sweep decides whether an unset `runtime.workers` runs one worker per core on Linux -- [An idle export poll is cheap](/quest/m2/perf/export-idle-poll.md) - an export poll nothing woke skips the track fill and catalog work it repeats today - [Mesh session cost](/quest/m2/perf/mesh-session-cost.md) - moq-net's per-session announce, route sync, and scope work drops, swept over sessions and announces - [Cluster route memory](/quest/m2/perf/cluster-route-memory.md) - a full-mesh cluster's ~3.7 KB per route per broadcast shrinks, the in-memory hop is profiled, and `bench-profile lock` tolerates lost records - [Lock-free single-delivery writes](/quest/m2/perf/single-delivery-write.md) - a frame with one delivery writes without a lock, recovering #6922's remaining FETCH cost diff --git a/quest/m2/perf/export-idle-poll.md b/quest/m2/perf/export-idle-poll.md deleted file mode 100644 index 08f46d9152a..00000000000 --- a/quest/m2/perf/export-idle-poll.md +++ /dev/null @@ -1,15 +0,0 @@ -# [M] An export poll nothing woke is cheap - -## Goal - -A container export poll that no new data woke costs little: today about 24% -of each poll is moq-net track fill and 11 to 19% is catalog polling. A -benchmark shows the idle poll's cost, before and after. - -## Plan - -Found by [moq-dev/moq#6987](https://github.com/moq-dev/moq/pull/6987)'s -profile of the moq-mux TS export tests. It helps those tests and production -exports alike. Profile first, then cut the work an idle poll repeats. - -Public API: none expected. diff --git a/quest/m2/perf/ts-idle-jitter.md b/quest/m2/perf/ts-idle-jitter.md index 1c06067e00c..166eed5b197 100644 --- a/quest/m2/perf/ts-idle-jitter.md +++ b/quest/m2/perf/ts-idle-jitter.md @@ -10,8 +10,5 @@ does, about 1.4k instructions per track. Today it is about 3k. Planned 2026-11-01 (maintainer, follow-up planning after the 2026-11-01 /quest-complete run). Found by [moq-dev/moq#7504](https://github.com/moq-dev/moq/pull/7504). With a delay, `jitter::Acquire::freshest` (18%) and `memcmp` (26%) lead the idle poll (`rs/moq-mux`). Skip the freshest scan when nothing arrived, and find what -the memcmp compares on an idle poll. Measure with #7504's export idle bench. - -## Required - -- [An export poll nothing woke is cheap](/quest/m2/perf/export-idle-poll.md) - adds the export idle bench and the baseline this measures against, in [moq-dev/moq#7504](https://github.com/moq-dev/moq/pull/7504) +the memcmp compares on an idle poll. Measure with the `container-export-idle` bench +(`rs/moq-mux/benches/export.rs`). diff --git a/rs/kio/src/waiter.rs b/rs/kio/src/waiter.rs index dda84522033..e2dc3bc5404 100644 --- a/rs/kio/src/waiter.rs +++ b/rs/kio/src/waiter.rs @@ -42,7 +42,8 @@ pub struct Waiter { // unrecorded and a later registration there would stack a duplicate. lost: AtomicBool, - // Made by `noop`, so a channel parks it nowhere: nothing would ever see its wake. + // Made by [`Self::noop`]: nothing would ever see its wake, so it registers nothing and a + // channel parks it nowhere. noop: bool, } @@ -256,7 +257,8 @@ impl WaiterList { } } - /// Register a waiter, a no-op while this list still holds it. + /// Register a waiter, a no-op while this list still holds it, or for a + /// [`Waiter::noop`]. /// /// The list's tag changes on every drain and the waiter records the tag it /// joined under, so a matching record proves the entry is still here and a @@ -267,6 +269,10 @@ impl WaiterList { /// window covers the list over time. A list about to grow sweeps every /// dead slot first. pub fn register(&mut self, waiter: &Waiter) { + // Registering would only allocate a handle nothing can wake, leaving a dead slot once it drops. + if waiter.noop { + return; + } if self.tag == 0 { self.tag = serial(); } @@ -579,7 +585,7 @@ mod tests { fn abandoned_waiters_do_not_grow_the_list() { let mut list = WaiterList::new(); for _ in 0..1_000 { - Waiter::noop().register(&mut list); + Waiter::new(Waker::noop().clone()).register(&mut list); } assert!(list.entries.len() <= 2); } @@ -588,13 +594,24 @@ mod tests { fn a_list_is_empty_once_its_waiters_drop() { let mut list = WaiterList::new(); assert!(list.is_empty()); - let waiter = Waiter::noop(); + let waiter = Waiter::new(Waker::noop().clone()); waiter.register(&mut list); assert!(!list.is_empty()); drop(waiter); assert!(list.is_empty()); } + /// A peek through a no-op waiter allocates no handle and takes no slot, clone or not. + #[test] + fn a_noop_waiter_registers_nothing() { + let mut list = WaiterList::new(); + let waiter = Waiter::noop(); + waiter.register(&mut list); + assert!(waiter.shared.get().is_none(), "a no-op registration allocated a handle"); + waiter.clone().register(&mut list); + assert!(list.entries.is_empty()); + } + /// Tasks parked on one list are retired and re-register in whatever order they /// wake, while the list itself is never woken (a rarely-changing value many tasks /// watch). The probe window alone let such a list grow without bound. @@ -602,7 +619,7 @@ mod tests { fn retired_live_waiters_do_not_grow_the_list() { const LIVE: usize = 64; let mut list = WaiterList::new(); - let mut waiters: Vec = (0..LIVE).map(|_| Waiter::noop()).collect(); + let mut waiters: Vec = (0..LIVE).map(|_| Waiter::new(Waker::noop().clone())).collect(); for waiter in &waiters { waiter.register(&mut list); } @@ -614,7 +631,7 @@ mod tests { for _ in 0..rounds { seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); let i = (seed >> 33) as usize % LIVE; - waiters[i] = Waiter::noop(); + waiters[i] = Waiter::new(Waker::noop().clone()); waiters[i].register(&mut list); } @@ -639,7 +656,7 @@ mod tests { /// after one of them drains, while an unrecorded list appends on every call. #[test] fn overflow_keeps_recorded_lists_idempotent() { - let waiter = Waiter::noop(); + let waiter = Waiter::new(Waker::noop().clone()); let mut lists: Vec<_> = (0..=PARKED).map(|_| WaiterList::new()).collect(); for _ in 0..3 { for list in &mut lists { diff --git a/rs/moq-mux/benches/export.rs b/rs/moq-mux/benches/export.rs index b88e8435952..db05677b0fc 100644 --- a/rs/moq-mux/benches/export.rs +++ b/rs/moq-mux/benches/export.rs @@ -1,4 +1,11 @@ -//! Container export cost swept over independently varied tracks and queued frames. +//! Container export cost swept over independently varied tracks and queued frames, and the +//! cost of a live export's poll that nothing new woke. +#![expect( + clippy::disallowed_methods, + clippy::disallowed_types, + reason = "`iter_custom` timings measure the host clock on purpose" +)] +use std::task::Poll; use std::time::Duration; use bytes::Bytes; @@ -22,8 +29,8 @@ impl Drop for Fixture { } } -/// A finished broadcast of `tracks` audio tracks with `frames` queued frames each. -async fn prepare(tracks: usize, frames: usize) -> (moq_mux::Source, moq_mux::catalog::Consumer, Fixture) { +/// A broadcast of `tracks` audio tracks with `frames` queued frames each, finished unless `live`. +async fn prepare(tracks: usize, frames: usize, live: bool) -> (moq_mux::Source, moq_mux::catalog::Consumer, Fixture) { let (origin, driver) = moq_net::origin::Producer::new(Default::default()); let driver = tokio::spawn(moq_net::time::run(driver)); let broadcast = origin.create_broadcast("bench").unwrap(); @@ -52,9 +59,13 @@ async fn prepare(tracks: usize, frames: usize) -> (moq_mux::Source, moq_mux::cat }) .unwrap(); } - track.finish().unwrap(); + if !live { + track.finish().unwrap(); + } + } + if !live { + catalog.finish().unwrap(); } - catalog.finish().unwrap(); broadcast.announce(Default::default()).unwrap(); let resolved = origin.consume().request("bench", None).await.unwrap(); let source = moq_mux::Source::new(origin.consume(), resolved); @@ -73,6 +84,17 @@ enum Export { Ts(Box>), } +impl Export { + /// Poll for the next chunk, ready with whether one came out rather than the export ending. + fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll { + match self { + Export::Flv(export) => export.poll_next(waiter).map(|res| res.unwrap().is_some()), + Export::Mkv(export) => export.poll_next(waiter).map(|res| res.unwrap().is_some()), + Export::Ts(export) => export.poll_next(waiter).map(|res| res.unwrap().is_some()), + } + } +} + fn open(format: &str, source: moq_mux::Source, stream: moq_mux::catalog::Consumer) -> Export { match format { "flv" => Export::Flv(Box::new( @@ -132,7 +154,7 @@ fn bench_export(c: &mut Criterion) { // Publishing and opening are setup; only draining the export is measured. b.iter_batched( || { - let (source, stream, fixture) = runtime.block_on(prepare(tracks, frames)); + let (source, stream, fixture) = runtime.block_on(prepare(tracks, frames, false)); (open(format, source, stream), fixture) }, |(opened, fixture)| { @@ -153,7 +175,50 @@ fn bench_export(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_export, video_reorder); +/// A live export polled again from the same task with nothing new: the poll a task pays for +/// whenever something else wakes it. Swept over tracks, a slope is work repeated per track. +fn export_idle(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let mut group = c.benchmark_group("container-export-idle"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(1)); + for format in ["flv", "mkv", "ts"] { + for tracks in [1, 4, 16] { + group.throughput(Throughput::Elements(tracks as u64)); + group.bench_function(BenchmarkId::new(format, tracks), |b| { + let (mut opened, _fixture) = runtime.block_on(async { + let (source, stream, fixture) = prepare(tracks, 4, true).await; + let mut opened = open(format, source, stream); + // Take what the export lets go now, giving the origin's tasks a turn between polls. + while let Ok(chunk) = + kio::clock::timeout(Duration::from_millis(10), kio::wait(|waiter| opened.poll_next(waiter))) + .await + { + assert!(chunk, "a live export ended"); + } + (opened, fixture) + }); + let mut park = kio::Park::default(); + b.iter_custom(|iterations| { + runtime.block_on(std::future::poll_fn(|cx| { + let start = std::time::Instant::now(); + for _ in 0..iterations { + // Each poll holds its waiter as `kio::wait` does. + assert!(std::hint::black_box(opened.poll_next(park.hold(cx))).is_pending()); + } + Poll::Ready(start.elapsed()) + })) + }); + }); + } + } + group.finish(); +} + +criterion_group!(benches, bench_export, video_reorder, export_idle); criterion_main!(benches); /// A progressive AVC SPS declaring a reorder depth of two (25 fps), as the catalog does. diff --git a/rs/moq-net/src/model/expiry.rs b/rs/moq-net/src/model/expiry.rs index 7ee947d8839..6a1d0b4966f 100644 --- a/rs/moq-net/src/model/expiry.rs +++ b/rs/moq-net/src/model/expiry.rs @@ -361,10 +361,15 @@ impl Wakes { mod test { use super::*; + /// A reader's waiter that stays registered while it lives: `kio::Waiter::noop` registers nothing. + fn parked() -> kio::Waiter { + kio::Waiter::new(std::task::Waker::noop().clone()) + } + #[test] fn landing_hint_keeps_both_indices_armed() { let wakes = Wakes::default(); - let waiter = kio::Waiter::noop(); + let waiter = parked(); assert!(!wakes.landing_waiters.load(Ordering::Relaxed)); wakes.watch_landing(0, &waiter); wakes.watch_held(10, &waiter); @@ -397,7 +402,7 @@ mod test { for sequence in 0..100 { // The last switch's reader gave its group up. drop(reader.take()); - let waiter = kio::Waiter::noop(); + let waiter = parked(); wakes.watch_held(sequence, &waiter); reader = Some(waiter); } @@ -416,7 +421,7 @@ mod test { let mut next = 0u64; let mut park = |count: usize| { for _ in 0..count { - let waiter = kio::Waiter::noop(); + let waiter = parked(); wakes.watch_deadline(Timestamp::from_micros(next).unwrap(), Duration::ZERO, &waiter); next += 1; } @@ -446,7 +451,7 @@ mod test { media_sweeps += usize::from(parked.deadlines.len() >= parked.deadlines_sweep); wall_sweeps += usize::from(parked.wall.len() >= parked.wall_sweep); } - let waiter = kio::Waiter::noop(); + let waiter = parked(); let budget = Duration::from_secs(1) + Duration::from_micros(i); wakes.watch_deadline(Timestamp::from_secs(10).unwrap(), budget, &waiter); wakes.watch_wall(now + budget, &waiter); @@ -465,7 +470,7 @@ mod test { let wakes = Wakes::default(); let now = crate::time::Instant::ZERO; for i in 0..1_000u64 { - let waiter = kio::Waiter::noop(); + let waiter = parked(); let budget = Duration::from_secs(1) + Duration::from_micros(i); wakes.watch_deadline(Timestamp::from_secs(10).unwrap(), budget, &waiter); wakes.watch_wall(now + budget, &waiter); diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index cd84338be76..8ad746efbdd 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -19,6 +19,7 @@ //! //! The stream is closed with [Error] when all writers or readers are dropped. use super::flate; +use super::idle::Idle; use super::pace::Pace; mod content; use crate::cache; @@ -2899,6 +2900,8 @@ impl Handle { adopted: false, stale_counted: Arc::default(), recover: None, + idle: None, + parked: false, arrived: self.alive.arrived, } } @@ -3340,6 +3343,11 @@ pub struct Consumer { // Handed out from a front's logical track: carries the read across route changes. // Boxed: it is the rare case. recover: Option>, + // The frame read that last parked, answered again without the read path until something + // it parked on fires, once a read parks twice in a row; see [`Self::poll_read_as`]. + idle: Option>>, + // The last frame read parked with no gate set up yet. + parked: bool, // When the group was created here. Kept across an adopted replacement copy, which the // group arrived before. arrived: crate::time::Instant, @@ -3428,6 +3436,9 @@ impl Clone for Consumer { adopted: self.adopted, stale_counted: self.stale_counted.clone(), recover: self.recover.clone(), + // A clone reads in parallel, so it parks for itself. + idle: None, + parked: false, arrived: self.arrived, } } @@ -3461,6 +3472,7 @@ impl Consumer { /// [`frame::Consumer::inflated`]), for egress that carries compression. A no-op on a /// plain group. pub(crate) fn compressed(mut self) -> Self { + Idle::disarm(&mut self.idle); self.cursor.raw = true; self.cursor.prefetch = Prefetch::default(); self @@ -3512,6 +3524,7 @@ impl Consumer { /// Keep applying this subscription's drift budget while the group is read. pub(super) fn with_expiry(mut self, expiry: Arc) -> Self { + Idle::disarm(&mut self.idle); self.expiry = Some(expiry); self } @@ -3534,6 +3547,7 @@ impl Consumer { /// Carry this group across a front's route changes; see [`super::resume`]. pub(super) fn with_recover(mut self, recover: super::resume::Recover) -> Self { + Idle::disarm(&mut self.idle); self.recover = Some(Box::new(recover)); self } @@ -3613,6 +3627,9 @@ impl Consumer { // Same frames by index, so only the channel changes; read progress and the // cap stay this cursor's. let (index, end, raw) = (self.cursor.index, self.cursor.end, self.cursor.raw); + // Another read, such as `poll_finished`, may move the cursor while a frame read + // is parked on the old copy. + Idle::disarm(&mut self.idle); recover.adopt(&replacement); self.expiry = self.expiry.as_ref().map(|expiry| expiry.for_track(&replacement.copy)); self.cursor = replacement.group.cursor; @@ -3669,6 +3686,8 @@ impl Consumer { expiry.is_expired(budget, waiter) }) { self.expired = true; + // Another read, such as `poll_expired`, may judge a frame read parked before it. + Idle::disarm(&mut self.idle); if !self.stale_counted.swap(true, Ordering::Relaxed) { self.stats.stale(self.cursor.unread_content()); } @@ -3733,6 +3752,7 @@ impl Consumer { /// reads past it end cleanly (`None`), as if the group finished there, and an omitted /// end removes the cap. Raising the cap re-offers frames that are still cached. pub fn with_range(mut self, frames: impl RangeBounds) -> Self { + Idle::disarm(&mut self.idle); let (start, end) = super::subscription::sequence_bounds(frames); self.cursor.start_at(start); self.cursor.end = end.map(|end| usize::try_from(end).unwrap_or(usize::MAX)); @@ -3742,6 +3762,7 @@ impl Consumer { /// Cap reads before frame `end`, or remove the cap with `None`, without moving the /// start, which [`Self::with_range`] would clamp up to the first available frame. pub(crate) fn with_end(mut self, end: impl Into>) -> Self { + Idle::disarm(&mut self.idle); self.cursor.end = end.into().map(|end| usize::try_from(end).unwrap_or(usize::MAX)); self } @@ -3752,6 +3773,7 @@ impl Consumer { /// never held. A [`Producer::start_at`] floor above `index` still surfaces as /// [`Error::Lagged`]. pub fn skip_to(&mut self, index: u64) { + Idle::disarm(&mut self.idle); self.cursor.skip_to(index); } @@ -3773,18 +3795,25 @@ impl Consumer { /// Returns None if the group is finished and the index is out of range, or the cursor /// passed the [`Self::with_range`] cap. pub fn poll_next_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { - if self.recover.is_none() { - let res = self.poll_next_frame_once(waiter); - return self.ended_on(res); - } - let res = self.poll_resumed(waiter, Self::poll_next_frame_once); - match (res, &self.recover) { - // A frame read from a copy that may fail too carries on the same way. - (Poll::Ready(Ok(Some(frame))), Some(recover)) => Poll::Ready(Ok(Some( - frame.with_recover((**recover).clone(), self.cursor.index as u64 - 1), - ))), - (res, _) => self.ended_on(res), + let res = match self.recover.is_none() { + true => { + let res = self.poll_next_frame_once(waiter); + self.ended_on(res) + } + false => match (self.poll_resumed(waiter, Self::poll_next_frame_once), &self.recover) { + // A frame read from a copy that may fail too carries on the same way. + (Poll::Ready(Ok(Some(frame))), Some(recover)) => Poll::Ready(Ok(Some( + frame.with_recover((**recover).clone(), self.cursor.index as u64 - 1), + ))), + (res, _) => self.ended_on(res), + }, + }; + // The cursor moved past a frame that may still be arriving, which a whole-frame read + // parked on it waits out: that read never hears of the move. + if let Poll::Ready(Ok(Some(_))) = &res { + Idle::disarm(&mut self.idle); } + res } /// Pass a read through, releasing the track demand once it reached the group's end, @@ -3837,18 +3866,56 @@ impl Consumer { } /// Read the next frame through the prefetch with `take`, which picks its form. + /// + /// A read that parked is answered again with `Pending` until something it parked on + /// fires: the group, its expiry, the reader's subscription, or the route. A read parks + /// only with its prefetch drained, so `take` does not change that answer. + /// + /// The gate is set up once a read parks twice in a row, as it does for a task that polls + /// it whenever another reader wakes, and arms on the read after: a group read whose next + /// poll is its own wake, as one task per group has it, never pays for one. fn poll_read_as( &mut self, waiter: &kio::Waiter, take: impl Fn(&mut Prefetch) -> Option + Copy, ) -> Poll>> { - let res = match self.recover.is_none() { - true => self.poll_read_frame_once(waiter, take), - false => self.poll_resumed(waiter, |this, waiter| this.poll_read_frame_once(waiter, take)), + if Idle::parked(&self.idle, (), waiter) { + return Poll::Pending; + } + let res = match self.idle.is_some() { + true => { + let mut idle = Idle::start(&mut self.idle, waiter); + let res = self.poll_read_frame_through(idle.waiter(), take); + idle.finish(&mut self.idle, (), res.is_pending()); + res + } + false => { + let res = self.poll_read_frame_through(waiter, take); + let parked = res.is_pending(); + if parked && self.parked { + // Parked twice in a row: the next read parks through a gate. Unarmed, since + // this read registered on the task's own waiter, not the gate's. + Idle::start(&mut self.idle, waiter).finish(&mut self.idle, (), false); + } + self.parked = parked; + res + } }; self.ended_on(res) } + /// Read the next frame, carried across route changes when the group came from a front. + fn poll_read_frame_through( + &mut self, + waiter: &kio::Waiter, + take: impl Fn(&mut Prefetch) -> Option + Copy, + ) -> Poll>> { + match self.recover.is_none() { + true => self.poll_read_frame_once(waiter, take), + false => self.poll_resumed(waiter, |this, waiter| this.poll_read_frame_once(waiter, take)), + } + } + /// Read the next frame with its relay metadata, for wire egress. #[cfg(feature = "async")] pub(crate) fn poll_read_object(&mut self, waiter: &kio::Waiter) -> Poll>> { @@ -6577,16 +6644,26 @@ mod test { assert!(matches!(res, Err(Error::ProtocolViolation))); } - /// A budget that gives up exactly when the test says so. - struct Switch(Arc); + /// A budget that gives up exactly when the test says so, waking the reads it judged as + /// a real budget does. + #[derive(Clone, Default)] + struct Switch(Arc<(AtomicBool, Mutex)>); + + impl Switch { + fn set(&self, expired: bool) { + self.0.0.store(expired, Ordering::Relaxed); + self.0.1.lock().unwrap().wake(); + } + } impl Expiry for Switch { - fn is_expired(&self, _: Option, _: &kio::Waiter) -> bool { - self.0.load(Ordering::Relaxed) + fn is_expired(&self, _: Option, waiter: &kio::Waiter) -> bool { + waiter.register(&mut self.0.1.lock().unwrap()); + self.0.0.load(Ordering::Relaxed) } fn for_track(&self, _: &track::Consumer) -> Arc { - Arc::new(Switch(self.0.clone())) + Arc::new(self.clone()) } } @@ -6600,13 +6677,14 @@ mod test { .write_frame(crate::Timestamp::ZERO, Bytes::from_static(b"a")) .unwrap(); - let expired = Arc::new(AtomicBool::new(false)); - let mut group = producer.consume().with_expiry(Arc::new(Switch(expired.clone()))); + let expired = Switch::default(); + let mut group = producer.consume().with_expiry(Arc::new(expired.clone())); let waiter = kio::Waiter::noop(); assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); assert!(group.poll_read_frame(&waiter).is_pending(), "parked at the tail"); + assert!(group.poll_read_frame(&waiter).is_pending(), "and parks again"); - expired.store(true, Ordering::Relaxed); + expired.set(true); let mut next = group.clone(); let mut finished = group.clone(); assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Err(Error::Old)))); @@ -6618,6 +6696,92 @@ mod test { assert!(matches!(finished.poll_finished(&waiter), Poll::Ready(Err(Error::Old)))); } + /// Records a wake. + #[derive(Default)] + struct Woken(AtomicBool); + + impl std::task::Wake for Woken { + fn wake(self: Arc) { + self.0.store(true, Ordering::Relaxed); + } + } + + impl Woken { + fn take(&self) -> bool { + self.0.swap(false, Ordering::Relaxed) + } + } + + /// A frame read that found nothing parks again at once until the group changes, but + /// parks anew for another task, which the next frame then wakes instead. + #[test] + fn an_idle_frame_read_parks_for_the_task_polling_it() { + let mut producer = Info { sequence: 0 }.produce(); + producer + .write_frame(crate::Timestamp::ZERO, Bytes::from_static(b"a")) + .unwrap(); + let mut group = producer.consume(); + let first = Arc::new(Woken::default()); + let waiter = kio::Waiter::new(first.clone().into()); + assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); + for _ in 0..3 { + assert!(group.poll_read_frame(&waiter).is_pending(), "nothing fired"); + } + assert!(!first.take()); + + let other = Arc::new(Woken::default()); + let moved = kio::Waiter::new(other.clone().into()); + assert!(group.poll_read_frame(&moved).is_pending()); + producer + .write_frame(crate::Timestamp::ZERO, Bytes::from_static(b"b")) + .unwrap(); + assert!(other.take(), "the next frame wakes the task that polled last"); + assert!(matches!(group.poll_read_frame(&moved), Poll::Ready(Ok(Some(_))))); + } + + /// A read that parks once and is then woken by its next frame never sets up a gate. + #[test] + fn a_read_woken_by_its_frame_takes_no_gate() { + let mut producer = Info { sequence: 0 }.produce(); + let mut group = producer.consume(); + let waiter = kio::Waiter::new(Arc::new(Woken::default()).into()); + for payload in [b"a", b"b", b"c"] { + assert!(group.poll_read_frame(&waiter).is_pending()); + producer + .write_frame(crate::Timestamp::ZERO, Bytes::from_static(payload)) + .unwrap(); + assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); + } + assert!(group.idle.is_none(), "a reader its own wake polls paid for a gate"); + } + + /// A chunked read that takes the frame a whole-frame read parked on moves the cursor, so + /// that read looks again rather than waiting out a frame it no longer reads. + #[test] + fn a_chunked_read_moves_a_parked_frame_read_on() { + let producer = Info { sequence: 0 }.produce(); + let mut group = producer.consume().with_range(0..1); + let mut frame = producer + .create_frame(frame::Info { + size: 6, + timestamp: Some(Timestamp::ZERO), + }) + .unwrap(); + frame.write(Bytes::from_static(b"foo")).unwrap(); + let waiter = kio::Waiter::noop(); + for _ in 0..3 { + assert!( + group.poll_read_frame(&waiter).is_pending(), + "the frame is still arriving" + ); + } + assert!(matches!(group.poll_next_frame(&waiter), Poll::Ready(Ok(Some(_))))); + assert!( + matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(None))), + "past the cap" + ); + } + /// A reader capped by its range finishes there cleanly whatever becomes of the frames past /// it, so a publisher that served the range keeps its stream when the rest fails upstream. #[test] @@ -6651,8 +6815,8 @@ mod test { .unwrap(); producer.finish().unwrap(); - let expired = Arc::new(AtomicBool::new(false)); - let mut group = producer.consume().with_expiry(Arc::new(Switch(expired.clone()))); + let expired = Switch::default(); + let mut group = producer.consume().with_expiry(Arc::new(expired.clone())); let waiter = kio::Waiter::noop(); assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); assert!( @@ -6661,9 +6825,9 @@ mod test { ); assert!(!group.poll_expired(&waiter)); - expired.store(true, Ordering::Relaxed); + expired.set(true); assert!(group.poll_expired(&waiter), "the writer abandons its stream"); - expired.store(false, Ordering::Relaxed); + expired.set(false); assert!(group.poll_expired(&waiter), "and the verdict stands"); } /// A captured job owns its release guard independently of the cached delivery. diff --git a/rs/moq-net/src/model/idle.rs b/rs/moq-net/src/model/idle.rs new file mode 100644 index 00000000000..441d8f6b773 --- /dev/null +++ b/rs/moq-net/src/model/idle.rs @@ -0,0 +1,35 @@ +//! The shared early exit for a read that found nothing. +//! +//! A read that parks registers on everything that can change its answer: for a group read, +//! the track, the subscription, the fills it waits on, the route, and the wall deadlines it +//! judged against; for a frame read, the group and its expiry. Each of those notifies on +//! every change, so until one fires, the answer stays the same. [`Idle`] answers a read +//! repeated before that wake with the same `Pending`, rather than walking the whole read +//! path again to reach it. +//! +//! A change the reader makes to its own state, such as moving its range, notifies nothing, +//! so it disarms the gate itself. +//! +//! Gates nest: a read through a gate that polls another gated reader passes it the outer +//! gate's waiter, whose waker stays the same across reads, so the inner gate keeps its arm +//! and its wake reaches the outer gate on the way to the task. + +pub(super) use crate::util::Idle; + +/// Which group read parked: a track reader parks the arrival and sequence walks +/// separately, since one finding nothing says nothing about the other. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum Walk { + Arrival, + Sequence, +} + +impl Walk { + /// The walk a read in sequence order for `ordered`, or arrival order otherwise, takes. + pub(super) fn new(ordered: bool) -> Self { + match ordered { + true => Self::Sequence, + false => Self::Arrival, + } + } +} diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index e0a67cc3cb5..d554e1c48e5 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -22,6 +22,8 @@ mod clock; pub(crate) mod datagram; mod delivered; mod expiry; +// The early exit a read nothing woke takes. +mod idle; mod skipped; // How far a group's writer reads ahead of its fastest reader. mod pace; diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index f2ce2682f5c..0fb11665f3a 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -51,6 +51,7 @@ use crate::time::Instant; use crate::{Error, Result, StreamError, group, stats, track}; use super::delivered::Delivered; +use super::idle::{Idle, Walk}; use super::subscription::{Accounted, Anchor, Position, Resolved, Subscription, Unheld, Window}; use crate::ledger; @@ -394,7 +395,11 @@ impl Consumer { .lock() .unwrap_or_else(PoisonError::into_inner) .sync(&kio::Waiter::noop()); - Subscriber { reader, subscription } + Subscriber { + reader, + subscription, + idle: None, + } } /// Fetch one group from whichever route serves the track. `logical` is the logical @@ -414,6 +419,12 @@ impl Consumer { } } + /// Register `waiter` for any change to the route, so a read that parked on a path that + /// never looked at the route still wakes when it changes. + fn register(&self, waiter: &kio::Waiter) { + let _ = self.state.poll(waiter, |_| Poll::<()>::Pending); + } + /// Poll for the route state to move past `generation`, or the track to end. fn poll_changed(&self, generation: u64, waiter: &kio::Waiter) -> Poll<()> { self.state @@ -706,6 +717,9 @@ pub(super) struct Subscriber { /// reader onto the next route without the reader polling for groups. reader: Arc>, subscription: track::Preferences, + /// The group read that last parked, answered again without the walk until something it + /// parked on fires, once it first parks; see [`super::idle`]. + idle: Option>>, } impl Subscriber { @@ -746,6 +760,12 @@ impl Subscriber { /// out: in arrival order, or in sequence order for `ordered`. /// Unless it `parks`, a bounded reader ends once its window has nothing left on any copy; /// see [`Reader::exhausted`]. + /// + /// A read that parked is answered again with `Pending` until something it parked on + /// fires: the reader's preferences, the route, a copy's answer, its cursor, or a wall + /// deadline. What the reader changes itself (its range, what it handed out) disarms it. + /// Another handle's read that follows the route or the preferences first only applies + /// a change that already woke this one. pub(super) fn poll_peek_group( &mut self, ordered: bool, @@ -753,12 +773,37 @@ impl Subscriber { delivered: &Delivered, waiter: &kio::Waiter, ) -> Poll>> { - let mut reader = self.reader(); - loop { - if let Some(result) = reader.poll_lowest(ordered, parks, delivered, waiter) { - return result; - } + let walk = Walk::new(ordered); + if self.parked(walk, waiter) { + return Poll::Pending; } + let mut idle = Idle::start(&mut self.idle, waiter); + let polled = { + let waiter = idle.waiter(); + let mut reader = self.reader(); + // Registered before looking, so a route change from here on wakes the read, + // even on a path that parks without looking at the route. + reader.route.register(waiter); + loop { + if let Some(result) = reader.poll_lowest(ordered, parks, delivered, waiter) { + break result; + } + } + }; + idle.finish(&mut self.idle, walk, polled.is_pending()); + polled + } + + /// Whether a group read in `walk` order would park again at once; see + /// [`Self::poll_peek_group`]. + pub(super) fn parked(&self, walk: Walk, waiter: &kio::Waiter) -> bool { + Idle::parked(&self.idle, walk, waiter) + } + + /// Forget the parked group read: what the reader handed out changed, which nothing + /// notifies. + pub(super) fn disarm(&mut self) { + Idle::disarm(&mut self.idle); } /// Whether this reader's bounded window has nothing left for the readers that handed out @@ -770,6 +815,7 @@ impl Subscriber { /// Hand out the group a [`Self::poll_peek_group`] reserved. pub(super) fn take(&mut self, candidate: Candidate) -> Result { + self.disarm(); self.reader().take(candidate) } @@ -851,6 +897,8 @@ impl Subscriber { /// Assign the local read floor and exclusive cap; see [`track::Subscriber::with_range`]. /// `delivered` says which groups a lowered floor newly owes. pub(super) fn set_range(&mut self, start: u64, end: Option, delivered: &Delivered) { + // A moved range notifies nothing a parked read waits on. + self.disarm(); self.reader().set_range(start, end, delivered) } @@ -2569,6 +2617,13 @@ mod test { Timestamp::from_millis(ms).unwrap() } + /// A task that has not polled yet. Nothing drives these copies' pools, so no wall + /// deadline wakes a read parked on one: once time passes, a test looks again from a + /// fresh task, since the same task's read answers as it parked until a wake. + fn fresh() -> kio::Waiter { + kio::Waiter::new(Arc::new(Wake::default()).into()) + } + /// A later stream header is not a verdict on the group held from a failed route. #[kio::test] async fn a_failed_route_waits_for_a_reordered_group_header() { @@ -2614,7 +2669,7 @@ mod test { } else { kio::clock::sleep(Duration::from_secs(1)).await; assert!(matches!( - held.poll_read_frame(&waiter), + held.poll_read_frame(&fresh()), Poll::Ready(Err(Error::NotFound)) )); } @@ -4377,7 +4432,10 @@ mod test { match outcome { "lands" => {} "dropped" => a.drop_groups(1..2, StreamError::NotFound), - _ => kio::clock::sleep(Duration::from_secs(1)).await, + _ => { + kio::clock::sleep(Duration::from_secs(1)).await; + assert!(sub.poll_recv_group(&fresh()).is_pending()); + } } if outcome != "lands" { assert_eq!(drain(&mut sub, false), Vec::::new()); @@ -4519,6 +4577,74 @@ mod test { assert!(matches!(sub.poll_recv_group(&noop), Poll::Ready(Ok(None)))); } + /// A read parked on that wait, which never looks at the route, still wakes once the + /// front serves another route. + #[test] + fn a_read_waiting_past_a_drained_copy_wakes_when_the_route_changes() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(1)); + write(&a, 0); + write(&a, 2); + assert_eq!(drain(&mut sub, false), [0, 2]); + + let b = copy(); + write(&b, 2); + b.finish().unwrap(); + routes.serve(b.consume()); + let wake = Arc::new(Wake::default()); + let waiter = kio::Waiter::new(wake.clone().into()); + assert!(sub.poll_recv_group(&waiter).is_pending(), "group 1 may still land"); + // Bounding the replaced copy updates its subscription, which wakes the read once. + wake.0.store(false, std::sync::atomic::Ordering::Relaxed); + assert!(sub.poll_recv_group(&waiter).is_pending()); + assert!( + sub.poll_recv_group(&waiter).is_pending(), + "nothing fired, so it parks again" + ); + assert!(!wake.0.load(std::sync::atomic::Ordering::Relaxed)); + + let c = copy(); + routes.serve(c.consume()); + assert!(wake.0.load(std::sync::atomic::Ordering::Relaxed)); + } + + /// A resumed read that found nothing parks again at once until something it parked on + /// fires, but parks anew for another task, which the next group then wakes instead. + #[test] + fn an_idle_resumed_read_parks_for_the_task_polling_it() { + for ordered in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(1)); + write(&a, 0); + let mut read = |waiter: &kio::Waiter| match ordered { + true => sub.poll_next_group(waiter), + false => sub.poll_recv_group(waiter), + }; + let first = Arc::new(Wake::default()); + let waiter = kio::Waiter::new(first.clone().into()); + assert!(matches!(read(&waiter), Poll::Ready(Ok(Some(_))))); + assert!(read(&waiter).is_pending()); + assert!(read(&waiter).is_pending(), "nothing fired, so it parks again"); + assert!(!first.0.load(std::sync::atomic::Ordering::Relaxed)); + + let other = Arc::new(Wake::default()); + let moved = kio::Waiter::new(other.clone().into()); + assert!(read(&moved).is_pending()); + write(&a, 1); + assert!( + other.0.load(std::sync::atomic::Ordering::Relaxed), + "the next group wakes the task that polled last" + ); + assert!(matches!(read(&moved), Poll::Ready(Ok(Some(_))))); + } + } + /// Datagrams are ephemeral: a reader that only reads them keeps no replaced copy, whatever /// its budget and whatever unread groups the copy holds, so a route that keeps moving /// leaves no copy per swap behind. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index c1f6b13ae63..a9ed0f02d11 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -19,6 +19,7 @@ use std::sync::Mutex; use super::Datagram; use super::delivered::Delivered; +use super::idle; use super::skipped::Skipped; use crate::ledger::{self, Ledger}; @@ -31,7 +32,6 @@ pub(super) use demand::Preferences; use demand::Subscriptions; mod fill; -mod idle; mod miss; pub use fill::Fill; use fill::FillState; @@ -6674,7 +6674,7 @@ struct Cursor { heads: Option>, /// The group read that last parked, answered again without the walk until something /// it parked on fires, once it first parks. - idle: Option>, + idle: Option>>, /// [`Self::state`] for [`Self::poll_finished`], which a publisher re-polls on every /// wake: one the track has not changed since skips the lock. Made on the first poll. finished: Option>>, @@ -8400,13 +8400,17 @@ impl Subscriber { /// already handed out. fn poll_group(&mut self, ordered: bool, waiter: &kio::Waiter) -> Poll>> { // Ahead of the meter: a read that parks again counts only what a peek skipped. - if let Inner::Plain(cursor) = &self.inner - && cursor.parked(idle::Walk::new(ordered), waiter) - { - if cursor.stale != stats::Content::default() && self.stats.is_tagged() { - self.count_stale(&self.stats.meter()); + let walk = idle::Walk::new(ordered); + match &self.inner { + Inner::Plain(cursor) if cursor.parked(walk, waiter) => { + if cursor.stale != stats::Content::default() && self.stats.is_tagged() { + self.count_stale(&self.stats.meter()); + } + return Poll::Pending; } - return Poll::Pending; + // A resumed read takes its copies' skips only on a full read, which counts them. + Inner::Resume(resume, _) if resume.parked(walk, waiter) => return Poll::Pending, + _ => {} } match self.poll_peek_group(ordered, waiter) { Poll::Ready(Ok(Some(reserved))) => Poll::Ready(self.take(reserved).map(Some)), @@ -8521,7 +8525,12 @@ impl Subscriber { pub(super) fn skip_ordered(&mut self, floor: Option) { match &mut self.inner { Inner::Plain(cursor) => cursor.skip_ordered(floor), - Inner::Resume(..) => { + Inner::Resume(resume, _) => { + // What it handed out moves, which nothing it parked on notifies. + let end = floor.unwrap_or(u64::MAX); + if !self.delivered.covers(0..end) || (floor.is_none() && !self.delivered.contains(u64::MAX)) { + resume.disarm(); + } // Its retention decisions also need the containing reader's past prefix. // This records what is no longer owed, without handing a group out. self.delivered.forget_below(floor.unwrap_or(u64::MAX)); diff --git a/rs/moq-net/src/model/track/idle.rs b/rs/moq-net/src/model/track/idle.rs deleted file mode 100644 index 05b05b5e020..00000000000 --- a/rs/moq-net/src/model/track/idle.rs +++ /dev/null @@ -1,31 +0,0 @@ -//! The shared early exit for a group read that found nothing. -//! -//! A read that parks registers on everything that can change its answer: the track, the -//! subscription, the fills it waits on, and the wall deadlines it judged against. Each of -//! those notifies on every change, so until one fires, the answer stays the same. [`Idle`] -//! answers a read repeated before that wake with the same `Pending`, rather than walking -//! the whole read path again to reach it. -//! -//! A change the reader makes to its own cursor, such as moving its range, notifies nothing, -//! so it disarms the gate itself. - -/// A cursor's parked group read, and the waiter it parked on. -pub(super) type Idle = crate::util::Idle; - -/// Which read parked: a [`Cursor`](super::Cursor) parks the arrival and sequence walks -/// separately, since one finding nothing says nothing about the other. -#[derive(Clone, Copy, PartialEq, Eq)] -pub(super) enum Walk { - Arrival, - Sequence, -} - -impl Walk { - /// The walk a read in sequence order for `ordered`, or arrival order otherwise, takes. - pub(super) fn new(ordered: bool) -> Self { - match ordered { - true => Self::Sequence, - false => Self::Arrival, - } - } -}