From 6760ef9f029bfb0b26ccca19132c4a70f761b500 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 23:35:15 -0700 Subject: [PATCH 1/8] chore(quest): claim quest/m2/perf/export-idle-poll.md Quest-Claim: 76f7b67d2577681c From 3103c0a33eefc8e7461921438e2f6b53c4ddc073 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 00:22:30 -0700 Subject: [PATCH 2/8] bench(mux): measure an idle export poll A live export polled again from the same task with nothing new, swept over formats and tracks, so work each idle poll repeats per track shows as a slope. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/benches/export.rs | 73 ++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 7 deletions(-) diff --git a/rs/moq-mux/benches/export.rs b/rs/moq-mux/benches/export.rs index b88e8435952..26f55dcf710 100644 --- a/rs/moq-mux/benches/export.rs +++ b/rs/moq-mux/benches/export.rs @@ -1,4 +1,6 @@ -//! Container export cost swept over independently varied tracks and queued frames. +//! Container export cost swept over independently varied tracks and queued frames, and the +//! cost of a live export's poll that nothing new woke. +use std::task::Poll; use std::time::Duration; use bytes::Bytes; @@ -22,8 +24,8 @@ impl Drop for Fixture { } } -/// A finished broadcast of `tracks` audio tracks with `frames` queued frames each. -async fn prepare(tracks: usize, frames: usize) -> (moq_mux::Source, moq_mux::catalog::Consumer, Fixture) { +/// A broadcast of `tracks` audio tracks with `frames` queued frames each, finished unless `live`. +async fn prepare(tracks: usize, frames: usize, live: bool) -> (moq_mux::Source, moq_mux::catalog::Consumer, Fixture) { let (origin, driver) = moq_net::origin::Producer::new(Default::default()); let driver = tokio::spawn(moq_net::time::run(driver)); let broadcast = origin.create_broadcast("bench").unwrap(); @@ -52,9 +54,13 @@ async fn prepare(tracks: usize, frames: usize) -> (moq_mux::Source, moq_mux::cat }) .unwrap(); } - track.finish().unwrap(); + if !live { + track.finish().unwrap(); + } + } + if !live { + catalog.finish().unwrap(); } - catalog.finish().unwrap(); broadcast.announce(Default::default()).unwrap(); let resolved = origin.consume().request("bench", None).await.unwrap(); let source = moq_mux::Source::new(origin.consume(), resolved); @@ -73,6 +79,17 @@ enum Export { Ts(Box>), } +impl Export { + /// Poll for the next chunk, ready with whether one came out rather than the export ending. + fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll { + match self { + Export::Flv(export) => export.poll_next(waiter).map(|res| res.unwrap().is_some()), + Export::Mkv(export) => export.poll_next(waiter).map(|res| res.unwrap().is_some()), + Export::Ts(export) => export.poll_next(waiter).map(|res| res.unwrap().is_some()), + } + } +} + fn open(format: &str, source: moq_mux::Source, stream: moq_mux::catalog::Consumer) -> Export { match format { "flv" => Export::Flv(Box::new( @@ -132,7 +149,7 @@ fn bench_export(c: &mut Criterion) { // Publishing and opening are setup; only draining the export is measured. b.iter_batched( || { - let (source, stream, fixture) = runtime.block_on(prepare(tracks, frames)); + let (source, stream, fixture) = runtime.block_on(prepare(tracks, frames, false)); (open(format, source, stream), fixture) }, |(opened, fixture)| { @@ -153,7 +170,49 @@ fn bench_export(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_export, video_reorder); +/// A live export polled again from the same task with nothing new: the poll a task pays for +/// whenever something else wakes it. Swept over tracks, a slope is work repeated per track. +fn export_idle(c: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let mut group = c.benchmark_group("container-export-idle"); + group.sample_size(10); + group.measurement_time(Duration::from_secs(1)); + for format in ["flv", "mkv", "ts"] { + for tracks in [1, 4, 16] { + group.throughput(Throughput::Elements(tracks as u64)); + group.bench_function(BenchmarkId::new(format, tracks), |b| { + let (mut opened, _fixture) = runtime.block_on(async { + let (source, stream, fixture) = prepare(tracks, 4, true).await; + let mut opened = open(format, source, stream); + // Take what the export lets go now, giving the origin's tasks a turn between polls. + while let Ok(chunk) = + tokio::time::timeout(Duration::from_millis(10), kio::wait(|waiter| opened.poll_next(waiter))).await + { + assert!(chunk, "a live export ended"); + } + (opened, fixture) + }); + let mut park = kio::Park::default(); + b.iter_custom(|iterations| { + runtime.block_on(std::future::poll_fn(|cx| { + let start = std::time::Instant::now(); + for _ in 0..iterations { + // Each poll holds its waiter as `kio::wait` does. + assert!(std::hint::black_box(opened.poll_next(park.hold(cx))).is_pending()); + } + Poll::Ready(start.elapsed()) + })) + }); + }); + } + } + group.finish(); +} + +criterion_group!(benches, bench_export, video_reorder, export_idle); criterion_main!(benches); /// A progressive AVC SPS declaring a reorder depth of two (25 fps), as the catalog does. From 5370d59609bc8ea40e548671014808ac02dac0c1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 00:22:30 -0700 Subject: [PATCH 3/8] perf(kio): a no-op waiter registers nothing Waiter::noop said it discards registrations, but registering one allocated its shared handle and filled a list slot that went dead as soon as it dropped. moq-net's resume layer peeks through one on every idle read. Co-Authored-By: Claude Opus 5.5 --- rs/kio/src/waiter.rs | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/rs/kio/src/waiter.rs b/rs/kio/src/waiter.rs index 27ea9769ae5..fb2429cbacf 100644 --- a/rs/kio/src/waiter.rs +++ b/rs/kio/src/waiter.rs @@ -41,6 +41,9 @@ pub struct Waiter { // A registration found no free slot in `parked`, so a list may hold this waiter // unrecorded and a later registration there would stack a duplicate. lost: AtomicBool, + + // Made by [`Self::noop`]: nothing it registers could ever be woken, so it registers nothing. + noop: bool, } /// Lists a waiter remembers being parked on. A relay's busiest tasks park on up to 8. @@ -54,12 +57,16 @@ impl Waiter { shared: OnceLock::new(), parked: Default::default(), lost: AtomicBool::new(false), + noop: false, } } /// Create a no-op waiter that discards registrations. pub fn noop() -> Self { - Self::new(Waker::noop().clone()) + Self { + noop: true, + ..Self::new(Waker::noop().clone()) + } } /// Register this waiter with a [`WaiterList`] for future notification. @@ -175,6 +182,7 @@ impl Clone for Waiter { shared: OnceLock::from(shared), parked: std::array::from_fn(|i| AtomicU64::new(self.parked[i].load(Ordering::Relaxed))), lost: AtomicBool::new(self.lost.load(Ordering::Relaxed)), + noop: self.noop, } } } @@ -232,7 +240,8 @@ impl WaiterList { } } - /// Register a waiter, a no-op while this list still holds it. + /// Register a waiter, a no-op while this list still holds it, or for a + /// [`Waiter::noop`]. /// /// The list's tag changes on every drain and the waiter records the tag it /// joined under, so a matching record proves the entry is still here and a @@ -243,6 +252,10 @@ impl WaiterList { /// window covers the list over time. A list about to grow sweeps every /// dead slot first. pub fn register(&mut self, waiter: &Waiter) { + // Registering would only allocate a handle nothing can wake, leaving a dead slot once it drops. + if waiter.noop { + return; + } if self.tag == 0 { self.tag = serial(); } @@ -546,7 +559,7 @@ mod tests { fn abandoned_waiters_do_not_grow_the_list() { let mut list = WaiterList::new(); for _ in 0..1_000 { - Waiter::noop().register(&mut list); + Waiter::new(Waker::noop().clone()).register(&mut list); } assert!(list.entries.len() <= 2); } @@ -555,13 +568,24 @@ mod tests { fn a_list_is_empty_once_its_waiters_drop() { let mut list = WaiterList::new(); assert!(list.is_empty()); - let waiter = Waiter::noop(); + let waiter = Waiter::new(Waker::noop().clone()); waiter.register(&mut list); assert!(!list.is_empty()); drop(waiter); assert!(list.is_empty()); } + /// A peek through a no-op waiter allocates no handle and takes no slot, clone or not. + #[test] + fn a_noop_waiter_registers_nothing() { + let mut list = WaiterList::new(); + let waiter = Waiter::noop(); + waiter.register(&mut list); + assert!(waiter.shared.get().is_none(), "a no-op registration allocated a handle"); + waiter.clone().register(&mut list); + assert!(list.entries.is_empty()); + } + /// Tasks parked on one list are retired and re-register in whatever order they /// wake, while the list itself is never woken (a rarely-changing value many tasks /// watch). The probe window alone let such a list grow without bound. @@ -569,7 +593,7 @@ mod tests { fn retired_live_waiters_do_not_grow_the_list() { const LIVE: usize = 64; let mut list = WaiterList::new(); - let mut waiters: Vec = (0..LIVE).map(|_| Waiter::noop()).collect(); + let mut waiters: Vec = (0..LIVE).map(|_| Waiter::new(Waker::noop().clone())).collect(); for waiter in &waiters { waiter.register(&mut list); } @@ -578,7 +602,7 @@ mod tests { for _ in 0..100_000 { seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); let i = (seed >> 33) as usize % LIVE; - waiters[i] = Waiter::noop(); + waiters[i] = Waiter::new(Waker::noop().clone()); waiters[i].register(&mut list); } @@ -603,7 +627,7 @@ mod tests { /// after one of them drains, while an unrecorded list appends on every call. #[test] fn overflow_keeps_recorded_lists_idempotent() { - let waiter = Waiter::noop(); + let waiter = Waiter::new(Waker::noop().clone()); let mut lists: Vec<_> = (0..=PARKED).map(|_| WaiterList::new()).collect(); for _ in 0..3 { for list in &mut lists { From 74865781f5b8cf8361c94fb30c4667c0a465dac8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 00:30:49 -0700 Subject: [PATCH 4/8] quest: rewrite export-idle-poll to the walk that remains Measured on top of moq-dev/moq#7263 and the no-op waiter fix: an idle export poll still costs about 5.7k to 10k instructions per track, mostly moq-net's resume reader and open group reads. Where to gate them is open. Also formats the idle export bench. Co-Authored-By: Claude Opus 5.5 --- quest/m2/perf/README.md | 2 +- quest/m2/perf/export-idle-poll.md | 35 +++++++++++++++++++++++++++---- rs/moq-mux/benches/export.rs | 3 ++- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/quest/m2/perf/README.md b/quest/m2/perf/README.md index 5086788bc2a..ac25070a22f 100644 --- a/quest/m2/perf/README.md +++ b/quest/m2/perf/README.md @@ -90,7 +90,7 @@ revisions on the same host and worker count, and profiles with - [Shard by default](/quest/m2/perf/shard-default.md) - a saturating shard sweep decides whether an unset `runtime.workers` runs one worker per core on Linux - [Profile the io_uring clock read](/quest/m2/perf/uring-clock-profile.md) - a `perf` profile shows whether #5926's per-turn clock read cut the io_uring relay's `[vdso]` share and CPU - [io_uring buffer pool memory](/quest/m2/perf/uring-pool-memory.md) - measure reserved versus resident pool memory before any shrink policy -- [An idle export poll is cheap](/quest/m2/perf/export-idle-poll.md) - an export poll nothing woke skips the track fill and catalog work it repeats today +- [An idle export poll is cheap](/quest/m2/perf/export-idle-poll.md) - an export poll nothing woke skips the per-track read walk it repeats today, through moq-net's resume reader and open group reads - [Mesh session cost](/quest/m2/perf/mesh-session-cost.md) - moq-net's per-session announce, route sync, and scope work drops, swept over sessions and announces - [Lite request allocations](/quest/m2/perf/lite-request-allocs.md) - `Path::concat` allocates once and the lite request handler keeps one track-name copy - [Coverage before START](/quest/m2/perf/covered-before-start.md) - the lite-07 publisher's coverage check costs O(1) per wake before SUBSCRIBE_START diff --git a/quest/m2/perf/export-idle-poll.md b/quest/m2/perf/export-idle-poll.md index 08f46d9152a..74998f4c3d4 100644 --- a/quest/m2/perf/export-idle-poll.md +++ b/quest/m2/perf/export-idle-poll.md @@ -2,14 +2,41 @@ ## Goal -A container export poll that no new data woke costs little: today about 24% -of each poll is moq-net track fill and 11 to 19% is catalog polling. A -benchmark shows the idle poll's cost, before and after. +A container export poll that no new data woke costs a small constant per +track, instead of walking every track's read path again to return `Pending`. +The `container-export-idle` group in `rs/moq-mux/benches/export.rs` shows the +cost, before and after. ## Plan Found by [moq-dev/moq#6987](https://github.com/moq-dev/moq/pull/6987)'s profile of the moq-mux TS export tests. It helps those tests and production -exports alike. Profile first, then cut the work an idle poll repeats. +exports alike. + +[moq-dev/moq#7263](https://github.com/moq-dev/moq/pull/7263) gates repeated +plain `Cursor` reads in moq-net (`track::idle`). Where the idle poll stands on +top of it, in user-space instructions per poll of a live audio export (pinned +core, `profiling` build), measured in +[moq-dev/moq#7504](https://github.com/moq-dev/moq/pull/7504): + +- About 5.7k to 10k per track, nearly linear in tracks (TS, 16 tracks: 110k). +- Catalog polling is now about 3%. +- About 45% is `track::Subscriber::poll_group` through `resume::Reader`. A + track served through an origin front is read as `Inner::Resume`, which the + `track::idle` gate skips. +- About a third is `group::Consumer::poll_read_frame` on the open group. Its + `poll_expiry` reaches `Recover::poll_budget`, which syncs the whole resume + reader on every pending read. +- The rest, about a fifth, is moq-mux bookkeeping per track. + +Open: where the early exit goes. + +- In moq-net (recommended): extend the `track::idle` gate to `Inner::Resume` + reads and to group frame reads. Every reader through a front gains. Each + input to the resume reader and `Recover` must notify, or disarm the gate. +- In moq-mux: one gate per export track around `container::Consumer`. Every + read below goes through it, so it skips the moq-mux bookkeeping too, but + nothing outside exports gains. State nothing notifies, such as a `Memory` + refund, must disarm it. Public API: none expected. diff --git a/rs/moq-mux/benches/export.rs b/rs/moq-mux/benches/export.rs index 26f55dcf710..461d3e685c7 100644 --- a/rs/moq-mux/benches/export.rs +++ b/rs/moq-mux/benches/export.rs @@ -189,7 +189,8 @@ fn export_idle(c: &mut Criterion) { let mut opened = open(format, source, stream); // Take what the export lets go now, giving the origin's tasks a turn between polls. while let Ok(chunk) = - tokio::time::timeout(Duration::from_millis(10), kio::wait(|waiter| opened.poll_next(waiter))).await + tokio::time::timeout(Duration::from_millis(10), kio::wait(|waiter| opened.poll_next(waiter))) + .await { assert!(chunk, "a live export ended"); } From 255515c08b4c078aed18f73bdac24c681828768b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 00:37:43 -0700 Subject: [PATCH 5/8] test(net): expiry sweeps park readers on waiters that register kio::Waiter::noop no longer registers, so it stood in for no reader at all. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/expiry.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/rs/moq-net/src/model/expiry.rs b/rs/moq-net/src/model/expiry.rs index 7ee947d8839..6a1d0b4966f 100644 --- a/rs/moq-net/src/model/expiry.rs +++ b/rs/moq-net/src/model/expiry.rs @@ -361,10 +361,15 @@ impl Wakes { mod test { use super::*; + /// A reader's waiter that stays registered while it lives: `kio::Waiter::noop` registers nothing. + fn parked() -> kio::Waiter { + kio::Waiter::new(std::task::Waker::noop().clone()) + } + #[test] fn landing_hint_keeps_both_indices_armed() { let wakes = Wakes::default(); - let waiter = kio::Waiter::noop(); + let waiter = parked(); assert!(!wakes.landing_waiters.load(Ordering::Relaxed)); wakes.watch_landing(0, &waiter); wakes.watch_held(10, &waiter); @@ -397,7 +402,7 @@ mod test { for sequence in 0..100 { // The last switch's reader gave its group up. drop(reader.take()); - let waiter = kio::Waiter::noop(); + let waiter = parked(); wakes.watch_held(sequence, &waiter); reader = Some(waiter); } @@ -416,7 +421,7 @@ mod test { let mut next = 0u64; let mut park = |count: usize| { for _ in 0..count { - let waiter = kio::Waiter::noop(); + let waiter = parked(); wakes.watch_deadline(Timestamp::from_micros(next).unwrap(), Duration::ZERO, &waiter); next += 1; } @@ -446,7 +451,7 @@ mod test { media_sweeps += usize::from(parked.deadlines.len() >= parked.deadlines_sweep); wall_sweeps += usize::from(parked.wall.len() >= parked.wall_sweep); } - let waiter = kio::Waiter::noop(); + let waiter = parked(); let budget = Duration::from_secs(1) + Duration::from_micros(i); wakes.watch_deadline(Timestamp::from_secs(10).unwrap(), budget, &waiter); wakes.watch_wall(now + budget, &waiter); @@ -465,7 +470,7 @@ mod test { let wakes = Wakes::default(); let now = crate::time::Instant::ZERO; for i in 0..1_000u64 { - let waiter = kio::Waiter::noop(); + let waiter = parked(); let budget = Duration::from_secs(1) + Duration::from_micros(i); wakes.watch_deadline(Timestamp::from_secs(10).unwrap(), budget, &waiter); wakes.watch_wall(now + budget, &waiter); From 965c742aafdf8c7e0bc3b2a1d1fcd9e4ef4a50a8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 15:13:52 -0800 Subject: [PATCH 6/8] perf(moq-net): an idle resumed read or frame read parks again without walking Extend the `track::idle` gate from plain cursors to the resume reader's group reads and to group frame reads, so a poll nothing woke answers `Pending` at once. - `idle.rs` moves up to `model/` and takes the read kind as a parameter. - `resume::Subscriber::poll_peek_group` parks through a gate. It registers on the route before looking, so a read that parks past a drained copy without looking at the route still wakes when it changes. A moved range and a moved handed-out floor (a nested route's `skip_ordered`) disarm it. - `group::Consumer` frame reads park through a gate, set up once a read parks twice in a row, so a group read woken only by its own data never allocates one. A moved range, a new expiry verdict, and an adopted replacement copy disarm it. - The export bench uses the ambient clock for its drain timeout and allows the host clock for `iter_custom`, as the other benches do. - Finishes `quest/m2/perf/export-idle-poll.md`, which this deletes. Co-Authored-By: Claude Opus 5.5 --- quest/m2/perf/README.md | 1 - quest/m2/perf/export-idle-poll.md | 42 ------- rs/moq-mux/benches/export.rs | 7 +- rs/moq-net/src/model/group.rs | 138 +++++++++++++++++++--- rs/moq-net/src/model/{track => }/idle.rs | 38 +++--- rs/moq-net/src/model/mod.rs | 2 + rs/moq-net/src/model/resume.rs | 142 +++++++++++++++++++++-- rs/moq-net/src/model/track.rs | 27 +++-- 8 files changed, 305 insertions(+), 92 deletions(-) delete mode 100644 quest/m2/perf/export-idle-poll.md rename rs/moq-net/src/model/{track => }/idle.rs (75%) diff --git a/quest/m2/perf/README.md b/quest/m2/perf/README.md index 7d077c64b63..a94c2fb00a9 100644 --- a/quest/m2/perf/README.md +++ b/quest/m2/perf/README.md @@ -94,7 +94,6 @@ revisions on the same host and worker count, and profiles with - [kio waiter costs](/quest/m2/perf/kio-waiter-costs.md) - long-lived waiters stop being rebuilt every poll, IETF fetch keeps one close watch per stream, and a short `kio::log` costs less - [Shard by default](/quest/m2/perf/shard-default.md) - a saturating shard sweep decides whether an unset `runtime.workers` runs one worker per core on Linux - [io_uring buffer pool memory](/quest/m2/perf/uring-pool-memory.md) - measure reserved versus resident pool memory before any shrink policy -- [An idle export poll is cheap](/quest/m2/perf/export-idle-poll.md) - an export poll nothing woke skips the per-track read walk it repeats today, through moq-net's resume reader and open group reads - [Mesh session cost](/quest/m2/perf/mesh-session-cost.md) - moq-net's per-session announce, route sync, and scope work drops, swept over sessions and announces - [Cluster route memory](/quest/m2/perf/cluster-route-memory.md) - a full-mesh cluster's ~3.7 KB per route per broadcast shrinks, the in-memory hop is profiled, and `bench-profile lock` tolerates lost records - [Adapter reads reuse their waker](/quest/m2/perf/adapter-waker-alloc.md) - a measured verdict on the per-read waker allocation moq-tokio and moq-uring reads pay since #7042, fixed if real diff --git a/quest/m2/perf/export-idle-poll.md b/quest/m2/perf/export-idle-poll.md deleted file mode 100644 index 74998f4c3d4..00000000000 --- a/quest/m2/perf/export-idle-poll.md +++ /dev/null @@ -1,42 +0,0 @@ -# [M] An export poll nothing woke is cheap - -## Goal - -A container export poll that no new data woke costs a small constant per -track, instead of walking every track's read path again to return `Pending`. -The `container-export-idle` group in `rs/moq-mux/benches/export.rs` shows the -cost, before and after. - -## Plan - -Found by [moq-dev/moq#6987](https://github.com/moq-dev/moq/pull/6987)'s -profile of the moq-mux TS export tests. It helps those tests and production -exports alike. - -[moq-dev/moq#7263](https://github.com/moq-dev/moq/pull/7263) gates repeated -plain `Cursor` reads in moq-net (`track::idle`). Where the idle poll stands on -top of it, in user-space instructions per poll of a live audio export (pinned -core, `profiling` build), measured in -[moq-dev/moq#7504](https://github.com/moq-dev/moq/pull/7504): - -- About 5.7k to 10k per track, nearly linear in tracks (TS, 16 tracks: 110k). -- Catalog polling is now about 3%. -- About 45% is `track::Subscriber::poll_group` through `resume::Reader`. A - track served through an origin front is read as `Inner::Resume`, which the - `track::idle` gate skips. -- About a third is `group::Consumer::poll_read_frame` on the open group. Its - `poll_expiry` reaches `Recover::poll_budget`, which syncs the whole resume - reader on every pending read. -- The rest, about a fifth, is moq-mux bookkeeping per track. - -Open: where the early exit goes. - -- In moq-net (recommended): extend the `track::idle` gate to `Inner::Resume` - reads and to group frame reads. Every reader through a front gains. Each - input to the resume reader and `Recover` must notify, or disarm the gate. -- In moq-mux: one gate per export track around `container::Consumer`. Every - read below goes through it, so it skips the moq-mux bookkeeping too, but - nothing outside exports gains. State nothing notifies, such as a `Memory` - refund, must disarm it. - -Public API: none expected. diff --git a/rs/moq-mux/benches/export.rs b/rs/moq-mux/benches/export.rs index 461d3e685c7..db05677b0fc 100644 --- a/rs/moq-mux/benches/export.rs +++ b/rs/moq-mux/benches/export.rs @@ -1,5 +1,10 @@ //! Container export cost swept over independently varied tracks and queued frames, and the //! cost of a live export's poll that nothing new woke. +#![expect( + clippy::disallowed_methods, + clippy::disallowed_types, + reason = "`iter_custom` timings measure the host clock on purpose" +)] use std::task::Poll; use std::time::Duration; @@ -189,7 +194,7 @@ fn export_idle(c: &mut Criterion) { let mut opened = open(format, source, stream); // Take what the export lets go now, giving the origin's tasks a turn between polls. while let Ok(chunk) = - tokio::time::timeout(Duration::from_millis(10), kio::wait(|waiter| opened.poll_next(waiter))) + kio::clock::timeout(Duration::from_millis(10), kio::wait(|waiter| opened.poll_next(waiter))) .await { assert!(chunk, "a live export ended"); diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index d66c6450ca9..f6c4a915b18 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -19,6 +19,7 @@ //! //! The stream is closed with [Error] when all writers or readers are dropped. use super::flate; +use super::idle::Idle; mod content; use crate::cache; use crate::frame::{self, Cached, FrameBuf, Payload}; @@ -2801,6 +2802,8 @@ impl Handle { adopted: false, stale_counted: Arc::default(), recover: None, + idle: None, + parked: false, arrived: self.alive.arrived, } } @@ -3226,6 +3229,11 @@ pub struct Consumer { // Handed out from a front's logical track: carries the read across route changes. // Boxed: it is the rare case. recover: Option>, + // The frame read that last parked, answered again without the read path until something + // it parked on fires, once a read parks twice in a row; see [`Self::poll_read_as`]. + idle: Option>>, + // The last frame read parked with no gate set up yet. + parked: bool, // When the group was created here. Kept across an adopted replacement copy, which the // group arrived before. arrived: crate::time::Instant, @@ -3313,6 +3321,9 @@ impl Clone for Consumer { adopted: self.adopted, stale_counted: self.stale_counted.clone(), recover: self.recover.clone(), + // A clone reads in parallel, so it parks for itself. + idle: None, + parked: false, arrived: self.arrived, } } @@ -3346,6 +3357,7 @@ impl Consumer { /// [`frame::Consumer::inflated`]), for egress that carries compression. A no-op on a /// plain group. pub(crate) fn compressed(mut self) -> Self { + Idle::disarm(&mut self.idle); self.cursor.raw = true; self.cursor.prefetch = Prefetch::default(); self @@ -3397,6 +3409,7 @@ impl Consumer { /// Keep applying this subscription's drift budget while the group is read. pub(super) fn with_expiry(mut self, expiry: Arc) -> Self { + Idle::disarm(&mut self.idle); self.expiry = Some(expiry); self } @@ -3419,6 +3432,7 @@ impl Consumer { /// Carry this group across a front's route changes; see [`super::resume`]. pub(super) fn with_recover(mut self, recover: super::resume::Recover) -> Self { + Idle::disarm(&mut self.idle); self.recover = Some(Box::new(recover)); self } @@ -3498,6 +3512,9 @@ impl Consumer { // Same frames by index, so only the channel changes; read progress and the // cap stay this cursor's. let (index, end, raw) = (self.cursor.index, self.cursor.end, self.cursor.raw); + // Another read, such as `poll_finished`, may move the cursor while a frame read + // is parked on the old copy. + Idle::disarm(&mut self.idle); recover.adopt(&replacement); self.expiry = self.expiry.as_ref().map(|expiry| expiry.for_track(&replacement.copy)); self.cursor = replacement.group.cursor; @@ -3554,6 +3571,8 @@ impl Consumer { expiry.is_expired(budget, waiter) }) { self.expired = true; + // Another read, such as `poll_expired`, may judge a frame read parked before it. + Idle::disarm(&mut self.idle); if !self.stale_counted.swap(true, Ordering::Relaxed) { self.stats.stale(self.cursor.unread_content()); } @@ -3618,6 +3637,7 @@ impl Consumer { /// reads past it end cleanly (`None`), as if the group finished there, and an omitted /// end removes the cap. Raising the cap re-offers frames that are still cached. pub fn with_range(mut self, frames: impl RangeBounds) -> Self { + Idle::disarm(&mut self.idle); let (start, end) = super::subscription::sequence_bounds(frames); self.cursor.start_at(start); self.cursor.end = end.map(|end| usize::try_from(end).unwrap_or(usize::MAX)); @@ -3627,6 +3647,7 @@ impl Consumer { /// Cap reads before frame `end`, or remove the cap with `None`, without moving the /// start, which [`Self::with_range`] would clamp up to the first available frame. pub(crate) fn with_end(mut self, end: impl Into>) -> Self { + Idle::disarm(&mut self.idle); self.cursor.end = end.into().map(|end| usize::try_from(end).unwrap_or(usize::MAX)); self } @@ -3637,6 +3658,7 @@ impl Consumer { /// never held. A [`Producer::start_at`] floor above `index` still surfaces as /// [`Error::Lagged`]. pub fn skip_to(&mut self, index: u64) { + Idle::disarm(&mut self.idle); self.cursor.skip_to(index); } @@ -3722,18 +3744,50 @@ impl Consumer { } /// Read the next frame through the prefetch with `take`, which picks its form. + /// + /// A read that parked is answered again with `Pending` until something it parked on + /// fires: the group, its expiry, the reader's subscription, or the route. A read parks + /// only with its prefetch drained, so `take` does not change that answer. + /// + /// The gate is set up once a read parks twice in a row, as it does for a task that polls + /// it whenever another reader wakes: a group read whose next poll is its own wake, as + /// one task per group has it, never pays for one. fn poll_read_as( &mut self, waiter: &kio::Waiter, take: impl Fn(&mut Prefetch) -> Option + Copy, ) -> Poll>> { - let res = match self.recover.is_none() { - true => self.poll_read_frame_once(waiter, take), - false => self.poll_resumed(waiter, |this, waiter| this.poll_read_frame_once(waiter, take)), + if Idle::parked(&self.idle, (), waiter) { + return Poll::Pending; + } + let res = match self.idle.is_some() || self.parked { + true => { + let mut idle = Idle::start(&mut self.idle, waiter); + let res = self.poll_read_frame_through(idle.waiter(), take); + idle.finish(&mut self.idle, (), res.is_pending()); + res + } + false => { + let res = self.poll_read_frame_through(waiter, take); + self.parked = res.is_pending(); + res + } }; self.ended_on(res) } + /// Read the next frame, carried across route changes when the group came from a front. + fn poll_read_frame_through( + &mut self, + waiter: &kio::Waiter, + take: impl Fn(&mut Prefetch) -> Option + Copy, + ) -> Poll>> { + match self.recover.is_none() { + true => self.poll_read_frame_once(waiter, take), + false => self.poll_resumed(waiter, |this, waiter| this.poll_read_frame_once(waiter, take)), + } + } + /// Read the next frame with its relay metadata, for wire egress. #[cfg(feature = "async")] pub(crate) fn poll_read_object(&mut self, waiter: &kio::Waiter) -> Poll>> { @@ -6449,16 +6503,26 @@ mod test { assert!(matches!(res, Err(Error::ProtocolViolation))); } - /// A budget that gives up exactly when the test says so. - struct Switch(Arc); + /// A budget that gives up exactly when the test says so, waking the reads it judged as + /// a real budget does. + #[derive(Clone, Default)] + struct Switch(Arc<(AtomicBool, Mutex)>); + + impl Switch { + fn set(&self, expired: bool) { + self.0.0.store(expired, Ordering::Relaxed); + self.0.1.lock().unwrap().wake(); + } + } impl Expiry for Switch { - fn is_expired(&self, _: Option, _: &kio::Waiter) -> bool { - self.0.load(Ordering::Relaxed) + fn is_expired(&self, _: Option, waiter: &kio::Waiter) -> bool { + waiter.register(&mut self.0.1.lock().unwrap()); + self.0.0.load(Ordering::Relaxed) } fn for_track(&self, _: &track::Consumer) -> Arc { - Arc::new(Switch(self.0.clone())) + Arc::new(self.clone()) } } @@ -6472,13 +6536,14 @@ mod test { .write_frame(crate::Timestamp::ZERO, Bytes::from_static(b"a")) .unwrap(); - let expired = Arc::new(AtomicBool::new(false)); - let mut group = producer.consume().with_expiry(Arc::new(Switch(expired.clone()))); + let expired = Switch::default(); + let mut group = producer.consume().with_expiry(Arc::new(expired.clone())); let waiter = kio::Waiter::noop(); assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); assert!(group.poll_read_frame(&waiter).is_pending(), "parked at the tail"); + assert!(group.poll_read_frame(&waiter).is_pending(), "and parks again"); - expired.store(true, Ordering::Relaxed); + expired.set(true); let mut next = group.clone(); let mut finished = group.clone(); assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Err(Error::Old)))); @@ -6490,6 +6555,49 @@ mod test { assert!(matches!(finished.poll_finished(&waiter), Poll::Ready(Err(Error::Old)))); } + /// Records a wake. + #[derive(Default)] + struct Woken(AtomicBool); + + impl std::task::Wake for Woken { + fn wake(self: Arc) { + self.0.store(true, Ordering::Relaxed); + } + } + + impl Woken { + fn take(&self) -> bool { + self.0.swap(false, Ordering::Relaxed) + } + } + + /// A frame read that found nothing parks again at once until the group changes, but + /// parks anew for another task, which the next frame then wakes instead. + #[test] + fn an_idle_frame_read_parks_for_the_task_polling_it() { + let mut producer = Info { sequence: 0 }.produce(); + producer + .write_frame(crate::Timestamp::ZERO, Bytes::from_static(b"a")) + .unwrap(); + let mut group = producer.consume(); + let first = Arc::new(Woken::default()); + let waiter = kio::Waiter::new(first.clone().into()); + assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); + for _ in 0..3 { + assert!(group.poll_read_frame(&waiter).is_pending(), "nothing fired"); + } + assert!(!first.take()); + + let other = Arc::new(Woken::default()); + let moved = kio::Waiter::new(other.clone().into()); + assert!(group.poll_read_frame(&moved).is_pending()); + producer + .write_frame(crate::Timestamp::ZERO, Bytes::from_static(b"b")) + .unwrap(); + assert!(other.take(), "the next frame wakes the task that polled last"); + assert!(matches!(group.poll_read_frame(&moved), Poll::Ready(Ok(Some(_))))); + } + /// A reader capped by its range finishes there cleanly whatever becomes of the frames past /// it, so a publisher that served the range keeps its stream when the rest fails upstream. #[test] @@ -6523,8 +6631,8 @@ mod test { .unwrap(); producer.finish().unwrap(); - let expired = Arc::new(AtomicBool::new(false)); - let mut group = producer.consume().with_expiry(Arc::new(Switch(expired.clone()))); + let expired = Switch::default(); + let mut group = producer.consume().with_expiry(Arc::new(expired.clone())); let waiter = kio::Waiter::noop(); assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); assert!( @@ -6533,9 +6641,9 @@ mod test { ); assert!(!group.poll_expired(&waiter)); - expired.store(true, Ordering::Relaxed); + expired.set(true); assert!(group.poll_expired(&waiter), "the writer abandons its stream"); - expired.store(false, Ordering::Relaxed); + expired.set(false); assert!(group.poll_expired(&waiter), "and the verdict stands"); } /// A captured job owns its release guard independently of the cached delivery. diff --git a/rs/moq-net/src/model/track/idle.rs b/rs/moq-net/src/model/idle.rs similarity index 75% rename from rs/moq-net/src/model/track/idle.rs rename to rs/moq-net/src/model/idle.rs index 21c0e02fb42..1f9a6a9980e 100644 --- a/rs/moq-net/src/model/track/idle.rs +++ b/rs/moq-net/src/model/idle.rs @@ -1,14 +1,19 @@ -//! The shared early exit for a group read that found nothing. +//! The shared early exit for a read that found nothing. //! -//! A read that parks registers on everything that can change its answer: the track, the -//! subscription, the fills it waits on, and the wall deadlines it judged against. Each of -//! those notifies on every change, so until one fires, the answer stays the same. [`Idle`] -//! parks reads through a waiter of its own that records any wake before passing it on to -//! the task, and answers a read repeated before that wake with the same `Pending`, rather -//! than walking the whole read path again to reach it. +//! A read that parks registers on everything that can change its answer: for a group read, +//! the track, the subscription, the fills it waits on, the route, and the wall deadlines it +//! judged against; for a frame read, the group and its expiry. Each of those notifies on +//! every change, so until one fires, the answer stays the same. [`Idle`] parks reads +//! through a waiter of its own that records any wake before passing it on to the task, and +//! answers a read repeated before that wake with the same `Pending`, rather than walking +//! the whole read path again to reach it. //! -//! Only what notifies can be trusted this way. A change the reader makes to its own cursor, +//! Only what notifies can be trusted this way. A change the reader makes to its own state, //! such as moving its range, notifies nothing, so it disarms the gate itself. +//! +//! Gates nest: a read through a gate that polls another gated reader passes it the outer +//! gate's waiter, whose waker stays the same across reads, so the inner gate keeps its arm +//! and its wake reaches the outer gate on the way to the task. use std::{ sync::{ @@ -18,7 +23,7 @@ use std::{ task::{Wake, Waker}, }; -/// Which read parked: a [`Cursor`](super::Cursor) parks the arrival and sequence walks +/// Which group read parked: a track reader parks the arrival and sequence walks /// separately, since one finding nothing says nothing about the other. #[derive(Clone, Copy, PartialEq, Eq)] pub(super) enum Walk { @@ -36,10 +41,11 @@ impl Walk { } } -/// A cursor's parked read, and the waiter it parked on. -pub(super) struct Idle { +/// A reader's parked read, and the waiter it parked on. `K` tells apart reads that park +/// separately, such as a [`Walk`]. +pub(super) struct Idle { /// The read that parked, or `None` while nothing is parked. - parked: Option, + parked: Option, woken: Arc, /// Wakes [`Self::woken`]. wake: Waker, @@ -68,7 +74,7 @@ impl Wake for Woken { } } -impl Idle { +impl Idle { fn new(task: &Waker) -> Self { let woken = Arc::new(Woken { woken: AtomicBool::new(false), @@ -84,7 +90,7 @@ impl Idle { /// Whether `walk`, read again for `task`, would park exactly as it last did: it parked, /// nothing it parked on has fired since, and it still wakes the same task. - pub(super) fn parked(this: &Option>, walk: Walk, task: &kio::Waiter) -> bool { + pub(super) fn parked(this: &Option>, walk: K, task: &kio::Waiter) -> bool { this.as_ref().is_some_and(|idle| { idle.parked == Some(walk) && !idle.woken.woken.load(Ordering::Acquire) @@ -114,12 +120,12 @@ impl Idle { } /// Put the gate back after a full read of `walk`, armed if it parked. - pub(super) fn finish(mut self: Box, this: &mut Option>, walk: Walk, pending: bool) { + pub(super) fn finish(mut self: Box, this: &mut Option>, walk: K, pending: bool) { self.parked = pending.then_some(walk); *this = Some(self); } - /// Forget the parked read: the cursor moved in a way nothing notifies. + /// Forget the parked read: the reader moved in a way nothing notifies. pub(super) fn disarm(this: &mut Option>) { if let Some(idle) = this { idle.parked = None; diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index 00a715a1efb..57408c04c71 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -22,6 +22,8 @@ mod clock; pub(crate) mod datagram; mod delivered; mod expiry; +// The early exit a read nothing woke takes. +mod idle; mod skipped; // The DEFLATE codec behind compressed tracks. mod flate; diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 5fbc5137807..89b54b53c4a 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -51,6 +51,7 @@ use crate::time::Instant; use crate::{Error, Result, StreamError, group, stats, track}; use super::delivered::Delivered; +use super::idle::{Idle, Walk}; use super::subscription::{Anchor, Position, Resolved, Subscription, Unheld, Window}; use crate::ledger; @@ -383,7 +384,11 @@ impl Consumer { .lock() .unwrap_or_else(PoisonError::into_inner) .sync(&kio::Waiter::noop()); - Subscriber { reader, subscription } + Subscriber { + reader, + subscription, + idle: None, + } } /// Fetch one group from whichever route serves the track. `logical` is the logical @@ -403,6 +408,12 @@ impl Consumer { } } + /// Register `waiter` for any change to the route, so a read that parked on a path that + /// never looked at the route still wakes when it changes. + fn watch(&self, waiter: &kio::Waiter) { + let _ = self.state.poll(waiter, |_| Poll::<()>::Pending); + } + /// Poll for the route state to move past `generation`, or the track to end. fn poll_changed(&self, generation: u64, waiter: &kio::Waiter) -> Poll<()> { match self.state.poll(waiter, |route| { @@ -671,6 +682,9 @@ pub(super) struct Subscriber { /// reader onto the next route without the reader polling for groups. reader: Arc>, subscription: track::Preferences, + /// The group read that last parked, answered again without the walk until something it + /// parked on fires, once it first parks; see [`super::idle`]. + idle: Option>>, } impl Subscriber { @@ -709,22 +723,54 @@ impl Mirror { impl Subscriber { /// Reserve the lowest next group not in `delivered` across copies, without handing it /// out: in arrival order, or in sequence order for `ordered`. + /// + /// A read that parked is answered again with `Pending` until something it parked on + /// fires: the reader's preferences, the route, a copy's answer, its cursor, or a wall + /// deadline. What the reader changes itself (its range, what it handed out) disarms it. + /// Another handle's read that follows the route or the preferences first only applies + /// a change that already woke this one. pub(super) fn poll_peek_group( &mut self, ordered: bool, delivered: &Delivered, waiter: &kio::Waiter, ) -> Poll>> { - let mut reader = self.reader(); - loop { - if let Some(result) = reader.poll_lowest(ordered, delivered, waiter) { - return result; - } + let walk = Walk::new(ordered); + if self.parked(walk, waiter) { + return Poll::Pending; } + let mut idle = Idle::start(&mut self.idle, waiter); + let polled = { + let waiter = idle.waiter(); + let mut reader = self.reader(); + // Registered before looking, so a route change from here on wakes the read, + // even on a path that parks without looking at the route. + reader.route.watch(waiter); + loop { + if let Some(result) = reader.poll_lowest(ordered, delivered, waiter) { + break result; + } + } + }; + idle.finish(&mut self.idle, walk, polled.is_pending()); + polled + } + + /// Whether a group read in `walk` order would park again at once; see + /// [`Self::poll_peek_group`]. + pub(super) fn parked(&self, walk: Walk, waiter: &kio::Waiter) -> bool { + Idle::parked(&self.idle, walk, waiter) + } + + /// Forget the parked group read: what the reader handed out changed, which nothing + /// notifies. + pub(super) fn disarm(&mut self) { + Idle::disarm(&mut self.idle); } /// Hand out the group a [`Self::poll_peek_group`] reserved. pub(super) fn take(&mut self, candidate: Candidate) -> Result { + self.disarm(); self.reader().take(candidate) } @@ -806,6 +852,8 @@ impl Subscriber { /// Assign the local read floor and exclusive cap; see [`track::Subscriber::with_range`]. /// `delivered` says which groups a lowered floor newly owes. pub(super) fn set_range(&mut self, start: u64, end: Option, delivered: &Delivered) { + // A moved range notifies nothing a parked read waits on. + self.disarm(); self.reader().set_range(start, end, delivered) } @@ -2456,6 +2504,13 @@ mod test { Timestamp::from_millis(ms).unwrap() } + /// A task that has not polled yet. Nothing drives these copies' pools, so no wall + /// deadline wakes a read parked on one: once time passes, a test looks again from a + /// fresh task, since the same task's read answers as it parked until a wake. + fn fresh() -> kio::Waiter { + kio::Waiter::new(Arc::new(Wake::default()).into()) + } + /// A later stream header is not a verdict on the group held from a failed route. #[kio::test] async fn a_failed_route_waits_for_a_reordered_group_header() { @@ -2501,7 +2556,7 @@ mod test { } else { kio::clock::sleep(Duration::from_secs(1)).await; assert!(matches!( - held.poll_read_frame(&waiter), + held.poll_read_frame(&fresh()), Poll::Ready(Err(Error::NotFound)) )); } @@ -4213,7 +4268,10 @@ mod test { match outcome { "lands" => {} "dropped" => a.drop_groups(1..2, StreamError::NotFound), - _ => kio::clock::sleep(Duration::from_secs(1)).await, + _ => { + kio::clock::sleep(Duration::from_secs(1)).await; + assert!(sub.poll_recv_group(&fresh()).is_pending()); + } } if outcome != "lands" { assert_eq!(drain(&mut sub, false), Vec::::new()); @@ -4355,6 +4413,74 @@ mod test { assert!(matches!(sub.poll_recv_group(&noop), Poll::Ready(Ok(None)))); } + /// A read parked on that wait, which never looks at the route, still wakes once the + /// front serves another route. + #[test] + fn a_read_waiting_past_a_drained_copy_wakes_when_the_route_changes() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(1)); + write(&a, 0); + write(&a, 2); + assert_eq!(drain(&mut sub, false), [0, 2]); + + let b = copy(); + write(&b, 2); + b.finish().unwrap(); + routes.serve(b.consume()); + let wake = Arc::new(Wake::default()); + let waiter = kio::Waiter::new(wake.clone().into()); + assert!(sub.poll_recv_group(&waiter).is_pending(), "group 1 may still land"); + // Bounding the replaced copy updates its subscription, which wakes the read once. + wake.0.store(false, std::sync::atomic::Ordering::Relaxed); + assert!(sub.poll_recv_group(&waiter).is_pending()); + assert!( + sub.poll_recv_group(&waiter).is_pending(), + "nothing fired, so it parks again" + ); + assert!(!wake.0.load(std::sync::atomic::Ordering::Relaxed)); + + let c = copy(); + routes.serve(c.consume()); + assert!(wake.0.load(std::sync::atomic::Ordering::Relaxed)); + } + + /// A resumed read that found nothing parks again at once until something it parked on + /// fires, but parks anew for another task, which the next group then wakes instead. + #[test] + fn an_idle_resumed_read_parks_for_the_task_polling_it() { + for ordered in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, Duration::from_secs(1)); + write(&a, 0); + let mut read = |waiter: &kio::Waiter| match ordered { + true => sub.poll_next_group(waiter), + false => sub.poll_recv_group(waiter), + }; + let first = Arc::new(Wake::default()); + let waiter = kio::Waiter::new(first.clone().into()); + assert!(matches!(read(&waiter), Poll::Ready(Ok(Some(_))))); + assert!(read(&waiter).is_pending()); + assert!(read(&waiter).is_pending(), "nothing fired, so it parks again"); + assert!(!first.0.load(std::sync::atomic::Ordering::Relaxed)); + + let other = Arc::new(Wake::default()); + let moved = kio::Waiter::new(other.clone().into()); + assert!(read(&moved).is_pending()); + write(&a, 1); + assert!( + other.0.load(std::sync::atomic::Ordering::Relaxed), + "the next group wakes the task that polled last" + ); + assert!(matches!(read(&moved), Poll::Ready(Ok(Some(_))))); + } + } + /// Datagrams are ephemeral: a reader that only reads them keeps no replaced copy, whatever /// its budget and whatever unread groups the copy holds, so a route that keeps moving /// leaves no copy per swap behind. diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 81315adec95..00f66dab7b8 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -19,6 +19,7 @@ use std::sync::Mutex; use super::Datagram; use super::delivered::Delivered; +use super::idle; use super::skipped::Skipped; use crate::ledger::{self, Ledger}; @@ -31,7 +32,6 @@ pub(super) use demand::Preferences; use demand::Subscriptions; mod fill; -mod idle; mod miss; pub use fill::Fill; use fill::FillState; @@ -6194,7 +6194,7 @@ struct Cursor { misses: Option>, /// The group read that last parked, answered again without the walk until something /// it parked on fires, once it first parks. - idle: Option>, + idle: Option>>, } /// A group a cursor found but has not moved past; see [`Cursor::take`]. @@ -7456,13 +7456,17 @@ impl Subscriber { /// already handed out. fn poll_group(&mut self, ordered: bool, waiter: &kio::Waiter) -> Poll>> { // Ahead of the meter: a read that parks again counts only what a peek skipped. - if let Inner::Plain(cursor) = &self.inner - && cursor.parked(idle::Walk::new(ordered), waiter) - { - if cursor.stale != stats::Content::default() && self.stats.is_tagged() { - self.count_stale(&self.stats.meter()); + let walk = idle::Walk::new(ordered); + match &self.inner { + Inner::Plain(cursor) if cursor.parked(walk, waiter) => { + if cursor.stale != stats::Content::default() && self.stats.is_tagged() { + self.count_stale(&self.stats.meter()); + } + return Poll::Pending; } - return Poll::Pending; + // A resumed read takes its copies' skips only on a full read, which counts them. + Inner::Resume(resume, _) if resume.parked(walk, waiter) => return Poll::Pending, + _ => {} } match self.poll_peek_group(ordered, waiter) { Poll::Ready(Ok(Some(reserved))) => Poll::Ready(self.take(reserved).map(Some)), @@ -7553,7 +7557,12 @@ impl Subscriber { pub(super) fn skip_ordered(&mut self, floor: Option) { match &mut self.inner { Inner::Plain(cursor) => cursor.skip_ordered(floor), - Inner::Resume(..) => { + Inner::Resume(resume, _) => { + // What it handed out moves, which nothing it parked on notifies. + let end = floor.unwrap_or(u64::MAX); + if !self.delivered.covers(0..end) || (floor.is_none() && !self.delivered.contains(u64::MAX)) { + resume.disarm(); + } // Its retention decisions also need the containing reader's past prefix. // This records what is no longer owed, without handing a group out. self.delivered.forget_below(floor.unwrap_or(u64::MAX)); From dcbc8073f3df5da53b63f0ebab6993b54e5cf17b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 15:17:48 -0800 Subject: [PATCH 7/8] fix(moq-net): a chunked read past a parked frame read disarms its gate `poll_next_frame` hands out a frame still arriving, which a whole-frame read waits out, so taking it moves the cursor without a wake. A capped reader whose parked `poll_read_frame` never heard of the move stayed `Pending` past its cap. Found by the local Codex review, round 1. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/group.rs | 56 ++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index f6c4a915b18..166794e0bca 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -3680,18 +3680,25 @@ impl Consumer { /// Returns None if the group is finished and the index is out of range, or the cursor /// passed the [`Self::with_range`] cap. pub fn poll_next_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { - if self.recover.is_none() { - let res = self.poll_next_frame_once(waiter); - return self.ended_on(res); - } - let res = self.poll_resumed(waiter, Self::poll_next_frame_once); - match (res, &self.recover) { - // A frame read from a copy that may fail too carries on the same way. - (Poll::Ready(Ok(Some(frame))), Some(recover)) => Poll::Ready(Ok(Some( - frame.with_recover((**recover).clone(), self.cursor.index as u64 - 1), - ))), - (res, _) => self.ended_on(res), + let res = match self.recover.is_none() { + true => { + let res = self.poll_next_frame_once(waiter); + self.ended_on(res) + } + false => match (self.poll_resumed(waiter, Self::poll_next_frame_once), &self.recover) { + // A frame read from a copy that may fail too carries on the same way. + (Poll::Ready(Ok(Some(frame))), Some(recover)) => Poll::Ready(Ok(Some( + frame.with_recover((**recover).clone(), self.cursor.index as u64 - 1), + ))), + (res, _) => self.ended_on(res), + }, + }; + // The cursor moved past a frame that may still be arriving, which a whole-frame read + // parked on it waits out: that read never hears of the move. + if let Poll::Ready(Ok(Some(_))) = &res { + Idle::disarm(&mut self.idle); } + res } /// Pass a read through, releasing the track demand once it reached the group's end, @@ -6598,6 +6605,33 @@ mod test { assert!(matches!(group.poll_read_frame(&moved), Poll::Ready(Ok(Some(_))))); } + /// A chunked read that takes the frame a whole-frame read parked on moves the cursor, so + /// that read looks again rather than waiting out a frame it no longer reads. + #[test] + fn a_chunked_read_moves_a_parked_frame_read_on() { + let producer = Info { sequence: 0 }.produce(); + let mut group = producer.consume().with_range(0..1); + let mut frame = producer + .create_frame(frame::Info { + size: 6, + timestamp: Some(Timestamp::ZERO), + }) + .unwrap(); + frame.write(Bytes::from_static(b"foo")).unwrap(); + let waiter = kio::Waiter::noop(); + for _ in 0..3 { + assert!( + group.poll_read_frame(&waiter).is_pending(), + "the frame is still arriving" + ); + } + assert!(matches!(group.poll_next_frame(&waiter), Poll::Ready(Ok(Some(_))))); + assert!( + matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(None))), + "past the cap" + ); + } + /// A reader capped by its range finishes there cleanly whatever becomes of the frames past /// it, so a publisher that served the range keeps its stream when the rest fails upstream. #[test] From 6526c064ac269070d45cbcd66a0447bc57febc51 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 22:27:33 -0800 Subject: [PATCH 8/8] fix(moq-net): set up a frame read's gate only once it parks twice in a row The gate was allocated on the read after one park, before that read knew whether it would park, so every live group reader that parked once paid for a gate for good. A read now sets the gate up, unarmed, only when it parks a second time in a row; the read after arms it. Regression test `a_read_woken_by_its_frame_takes_no_gate`. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/group.rs | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index 6dc25024bc8..8ad746efbdd 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -3872,8 +3872,8 @@ impl Consumer { /// only with its prefetch drained, so `take` does not change that answer. /// /// The gate is set up once a read parks twice in a row, as it does for a task that polls - /// it whenever another reader wakes: a group read whose next poll is its own wake, as - /// one task per group has it, never pays for one. + /// it whenever another reader wakes, and arms on the read after: a group read whose next + /// poll is its own wake, as one task per group has it, never pays for one. fn poll_read_as( &mut self, waiter: &kio::Waiter, @@ -3882,7 +3882,7 @@ impl Consumer { if Idle::parked(&self.idle, (), waiter) { return Poll::Pending; } - let res = match self.idle.is_some() || self.parked { + let res = match self.idle.is_some() { true => { let mut idle = Idle::start(&mut self.idle, waiter); let res = self.poll_read_frame_through(idle.waiter(), take); @@ -3891,7 +3891,13 @@ impl Consumer { } false => { let res = self.poll_read_frame_through(waiter, take); - self.parked = res.is_pending(); + let parked = res.is_pending(); + if parked && self.parked { + // Parked twice in a row: the next read parks through a gate. Unarmed, since + // this read registered on the task's own waiter, not the gate's. + Idle::start(&mut self.idle, waiter).finish(&mut self.idle, (), false); + } + self.parked = parked; res } }; @@ -6733,6 +6739,22 @@ mod test { assert!(matches!(group.poll_read_frame(&moved), Poll::Ready(Ok(Some(_))))); } + /// A read that parks once and is then woken by its next frame never sets up a gate. + #[test] + fn a_read_woken_by_its_frame_takes_no_gate() { + let mut producer = Info { sequence: 0 }.produce(); + let mut group = producer.consume(); + let waiter = kio::Waiter::new(Arc::new(Woken::default()).into()); + for payload in [b"a", b"b", b"c"] { + assert!(group.poll_read_frame(&waiter).is_pending()); + producer + .write_frame(crate::Timestamp::ZERO, Bytes::from_static(payload)) + .unwrap(); + assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); + } + assert!(group.idle.is_none(), "a reader its own wake polls paid for a gate"); + } + /// A chunked read that takes the frame a whole-frame read parked on moves the cursor, so /// that read looks again rather than waiting out a frame it no longer reads. #[test]