diff --git a/quest/m1/cut-audit/README.md b/quest/m1/cut-audit/README.md index d14c29bcbbc..16293a72596 100644 --- a/quest/m1/cut-audit/README.md +++ b/quest/m1/cut-audit/README.md @@ -54,7 +54,6 @@ leaving JS-only redesigns of `js/net` to [rs2ts](/quest/m2/rs2ts-js.md). ## Required -- [A reserved group survives a route returning](/quest/m1/cut-audit/reserved-route-return.md) - a route flip A to B to A no longer panics a peeked reader - [Stream-accept loops yield on a per-poll budget](/quest/m1/cut-audit/accept-budget.md) - a peer opening many streams cannot monopolize a session driver - [Rust and JS skip a backlog by the same measure](/quest/m1/cut-audit/catalog-parity.md) - a backlog join starts at the same group in every player - [Text and data sections keep a prototype-named ID](/quest/m1/cut-audit/catalog-proto-ids.md) - text and data entries keep the ID JS's `z.record` drops diff --git a/quest/m1/cut-audit/reserved-route-return.md b/quest/m1/cut-audit/reserved-route-return.md deleted file mode 100644 index 1d8f8936d77..00000000000 --- a/quest/m1/cut-audit/reserved-route-return.md +++ /dev/null @@ -1,42 +0,0 @@ -# [S] A reserved group survives a route returning to an earlier copy - -## Goal - -A resumed reader that peeked a group keeps that reservation when its route -flips A to B and back to A, and its next take returns the reserved group -instead of panicking at `resume.rs`'s `unreachable!("the peek reserved a -group")`. - -## Plan - -Found by the 2026-10-24 pre-cut audit at 151a529670. Reproduced, new on main -(release's splice design has no peek path). - -A reservation lives in two places: `Reader.peeked` (a `Candidate`) and the -copy cursor's `Cursor.peeked`. On a return to copy A, `Reader::sync` -(`rs/moq-net/src/model/resume.rs:958-990`) reuses A's copy, raises its floor, -and calls `set_range` (`model/track.rs:7013`), which clears the cursor's -reservation while `Reader.peeked` survives. If the reserved group was evicted -meanwhile, or sits below the new floor, the take hits `unreachable!` (about -`resume.rs:1250`) or the `assert_eq!` after it. In production the peek path -runs when a route copy is itself a front's logical track (origin-to-origin -chaining), and the flip must land between peek and take; the panic is -contained to the session by b685511410. - -The audit's reproduction, a unit test for `resume.rs`'s test module: build a -nested logical track, write group 0 on copy A and drain it, write and finish -group 2, `poll_peek_group` it, abort group 2 with `Error::Evicted`, serve -copy B, serve copy A again, then take with both `poll_next_group` and -`poll_recv_group`. It panics in both orders. - -Fix direction: the `Candidate` carries the reserved `group::Consumer` itself, -plus a cursor step that commits the sequence, so a take never re-queries a -cursor that `set_range` or `update` may have reset. That deletes the -`unreachable!`/`assert_eq!` pair. The narrower alternative is a `set_range` -that keeps a reservation still inside its new bounds. - -Public API: none. Wire: none. - -## Related - -- [Resume as producer replacement](/quest/m2/resume-redesign.md) - the redesign that deletes the two-level reservation diff --git a/quest/m2/resume-redesign.md b/quest/m2/resume-redesign.md index 6bd392b3514..b18be7f2520 100644 --- a/quest/m2/resume-redesign.md +++ b/quest/m2/resume-redesign.md @@ -87,5 +87,4 @@ Public API: none. Wire: none. ## Related -- [A reserved group survives a route returning](/quest/m1/cut-audit/reserved-route-return.md) - a pre-cut fix this deletes - [Recover reads its budget without a sync](/quest/m2/recover-budget.md) - a per-frame cost this deletes diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 48b75e61e5b..556447e9721 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -362,7 +362,6 @@ impl Consumer { groups_read: false, out: Vec::new(), drops: RangeMap::new(), - peeked: None, declared: None, declared_error: None, refused: false, @@ -708,49 +707,25 @@ impl Mirror { } impl Subscriber { - /// Poll for the next group not in `delivered`: in arrival order, or in sequence order - /// for `ordered`. - pub(super) fn poll_group( - &mut self, - ordered: bool, - delivered: &Delivered, - waiter: &kio::Waiter, - ) -> Poll>> { - self.reader().poll_group(ordered, delivered, waiter) - } - - /// The lowest next group across copies, without handing it out. + /// Reserve the lowest next group not in `delivered` across copies, without handing it + /// out: in arrival order, or in sequence order for `ordered`. pub(super) fn poll_peek_group( &mut self, ordered: bool, delivered: &Delivered, waiter: &kio::Waiter, - ) -> Poll>> { + ) -> Poll>> { let mut reader = self.reader(); - reader.peeked = None; loop { if let Some(result) = reader.poll_lowest(ordered, delivered, waiter) { - return result.map(|result| { - result.map(|next| { - let sequence = next.as_ref().map(|next| next.sequence); - reader.peeked = next; - sequence - }) - }); + return result; } } } - /// Invalidate a reservation below the containing reader's past prefix. - pub(super) fn skip_ordered(&mut self, floor: Option) { - let mut reader = self.reader(); - if reader - .peeked - .as_ref() - .is_some_and(|peeked| floor.is_none_or(|floor| peeked.sequence < floor)) - { - reader.peeked = None; - } + /// Hand out the group a [`Self::poll_peek_group`] reserved. + pub(super) fn take(&mut self, candidate: Candidate) -> Result { + self.reader().take(candidate) } /// Poll for the next datagram in arrival order. `delivered` decides which replaced @@ -920,12 +895,25 @@ impl Subscriber { } } -struct Candidate { - sequence: u64, - /// Its copy's lease, which names that copy: one route can back several. A nested reader - /// may sync a route swap before its caller consumes this candidate, so it also keeps - /// that exact copy alive until that commit. +/// A group a [`Reader`] reserved on one of its copies. +pub(super) struct Candidate { + /// Its copy's lease, which names that copy: one route can back several. A route may + /// change before the caller takes this candidate, so it also keeps that exact copy + /// alive until then. lease: Arc<()>, + /// The copy's own reservation, which taking hands out whatever the copy's cursor did + /// since: a route returning to it moves its bounds. + reserved: track::Reserved, +} + +impl Candidate { + pub(super) fn sequence(&self) -> u64 { + self.reserved.sequence() + } + + pub(super) fn ordered(&self) -> bool { + self.reserved.ordered() + } } /// Exact identities within the reader's budget. The last delivery never ages until @@ -972,7 +960,6 @@ impl Datagrams { /// A [`Subscriber`]'s state. struct Reader { - peeked: Option, /// The first exclusive end any copy declared, refined by a matching mid-group end. declared: Option, /// The first error a copy's end carried, kept with the end so a route switch keeps it. @@ -1262,9 +1249,6 @@ impl Reader { self.refused = true; } } - if self.refused { - self.peeked = None; - } } /// Whether `at` is past the declared end, which refuses the track: another copy @@ -1272,7 +1256,6 @@ impl Reader { fn past_end(&mut self, at: Position) -> bool { if self.declared.is_some_and(|end| at >= end) { self.refused = true; - self.peeked = None; } self.refused } @@ -1398,38 +1381,12 @@ impl Reader { }) } - fn poll_group( - &mut self, - ordered: bool, - delivered: &Delivered, - waiter: &kio::Waiter, - ) -> Poll>> { - loop { - if let Some(res) = self.poll_group_once(ordered, delivered, waiter) { - return res; - } - } - } - - /// One pass over the copies; `None` when the route moved meanwhile and it is worth - /// another. - fn poll_group_once( - &mut self, - ordered: bool, - delivered: &Delivered, - waiter: &kio::Waiter, - ) -> Option>>> { - let candidate = match self.peeked.take() { - Some(candidate) => candidate, - None => match self.poll_lowest(ordered, delivered, waiter)? { - Poll::Ready(Ok(Some(next))) => next, - Poll::Ready(Ok(None)) => return Some(Poll::Ready(Ok(None))), - Poll::Ready(Err(err)) => return Some(Poll::Ready(Err(err))), - Poll::Pending => return Some(Poll::Pending), - }, - }; - if self.past_end(Position::group(candidate.sequence)) { - return Some(Poll::Ready(Err(Error::ProtocolViolation))); + /// Hand out the group `candidate` reserved. Selection reserves the exact group, so + /// eviction can abort that group but cannot substitute a later one and advance the + /// logical reader past a gap. + fn take(&mut self, candidate: Candidate) -> Result { + if self.past_end(Position::group(candidate.sequence())) { + return Err(Error::ProtocolViolation); } let index = self .copies @@ -1437,18 +1394,9 @@ impl Reader { .position(|copy| Arc::ptr_eq(©.lease, &candidate.lease)) .expect("the candidate's lease retains its copy"); let Sub::Ready(sub) = &mut self.copies[index].sub else { - unreachable!() - }; - let result = match ordered { - true => sub.poll_next_group(waiter), - false => sub.poll_recv_group(waiter), - }; - // Selection reserves the exact group, so eviction can abort that group but - // cannot substitute a later one and advance the logical reader past a gap. - let Poll::Ready(Ok(Some(mut group))) = result else { - unreachable!("the peek reserved a group") + unreachable!("a candidate comes from a copy that answered") }; - assert_eq!(group.sequence, candidate.sequence, "the peek reserved this exact group"); + let mut group = sub.take(candidate.reserved)?; self.observe(index, group.sequence); // A mid-group end caps the group it falls in, as it caps a direct track's. if let Some(end) = self @@ -1457,8 +1405,7 @@ impl Reader { { group = group.with_end(end.frame); } - let group = self.hand_out(index, group); - Some(Poll::Ready(Ok(Some(group)))) + Ok(self.hand_out(index, group)) } /// Choose without consuming a candidate, so a newer copy cannot overtake a lower @@ -1517,7 +1464,8 @@ impl Reader { let res = sub.poll_peek_group(ordered, waiter); match res { Poll::Pending => break, - Poll::Ready(Ok(Some(sequence))) => { + Poll::Ready(Ok(Some(reserved))) => { + let sequence = reserved.sequence(); if !Self::deliverable(&self.copies[index], sequence, delivered) { let Sub::Ready(sub) = &mut self.copies[index].sub else { unreachable!() @@ -1525,14 +1473,14 @@ impl Reader { if ordered { sub.skip_ordered(sequence.checked_add(1)); } else { - let _ = sub.poll_recv_group(waiter); + let _ = sub.take(reserved); } continue; } - if lowest.as_ref().is_none_or(|lowest| sequence <= lowest.sequence) { + if lowest.as_ref().is_none_or(|lowest| sequence <= lowest.sequence()) { lowest = Some(Candidate { - sequence, lease: self.copies[index].lease.clone(), + reserved, }); } break; @@ -1826,7 +1774,6 @@ impl Reader { /// Assign the local floor and cap. A copy still waiting on its info takes them when it /// resolves; one already reading has to be moved, or a group it skipped stays skipped. fn set_range(&mut self, start: u64, end: Option, delivered: &Delivered) { - self.peeked = None; let passed = std::mem::replace(&mut self.groups, (start, end)).0; for copy in &mut self.copies { let start = copy.start(start); @@ -3905,6 +3852,14 @@ mod test { group.finish().unwrap(); } + /// The group a peek reserves, to take later. + fn peek(sub: &mut track::Subscriber, ordered: bool) -> track::Reserved { + let Poll::Ready(Ok(Some(reserved))) = sub.poll_peek_group(ordered, &kio::Waiter::noop()) else { + panic!("a group to reserve") + }; + reserved + } + /// Every group the reader hands out right now. fn drain(sub: &mut track::Subscriber, ordered: bool) -> Vec { let mut sequences = Vec::new(); @@ -4121,28 +4076,65 @@ mod test { one.write_frame(ts(1), b"x".as_ref()).unwrap(); one.finish().unwrap(); write(&a, 3); - let waiter = kio::Waiter::noop(); - assert_eq!( - sub.poll_peek_group(ordered, &waiter).map(Result::unwrap), - Poll::Ready(Some(1)) - ); + let reserved = peek(&mut sub, ordered); + assert_eq!(reserved.sequence(), 1); one.abort(Error::Evicted).unwrap(); let b = copy(); routes.serve(b.consume()); write(&b, 2); - let next = match ordered { - true => sub.poll_next_group(&waiter), - false => sub.poll_recv_group(&waiter), - }; - let Poll::Ready(Ok(Some(group))) = next else { - panic!("the reserved group") - }; + let group = sub.take(reserved).unwrap(); assert_eq!(group.sequence, 1, "ordered={ordered}, nested={nested}"); assert_eq!(drain(&mut sub, ordered), [2], "ordered={ordered}, nested={nested}"); } } } + /// A route that returns to the reserving copy after another served moves that copy's + /// bounds, even past the reserved group when it arrived late, but taking still hands it + /// out: whichever reader's route returns, the containing one's or a nested copy's. + #[test] + fn a_reserved_group_survives_its_route_returning() { + for ordered in [false, true] { + for late in [false, true].into_iter().filter(|late| !(ordered && *late)) { + for (nested, outer_flips) in [(false, false), (true, false), (true, true)] { + let case = format!("ordered={ordered}, late={late}, nested={nested}, outer_flips={outer_flips}"); + let routes = Producer::new(); + let inner = logical(&routes); + let outer_routes = Producer::new(); + let outer = logical(&outer_routes); + outer_routes.serve(inner.consume()); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(if nested { &outer } else { &inner }, HOUR); + write(&a, 0); + if late { + write(&a, 3); + } + drain(&mut sub, ordered); + let mut two = a.create_group(group::Info { sequence: 2 }).unwrap(); + two.write_frame(ts(2), b"x".as_ref()).unwrap(); + two.finish().unwrap(); + let reserved = peek(&mut sub, ordered); + assert_eq!(reserved.sequence(), 2, "{case}"); + two.abort(Error::Evicted).unwrap(); + match outer_flips { + true => { + outer_routes.serve(copy().consume()); + outer_routes.serve(inner.consume()); + } + false => { + routes.serve(copy().consume()); + routes.serve(a.consume()); + } + } + assert_eq!(sub.take(reserved).unwrap().sequence, 2, "{case}"); + write(&a, 4); + assert_eq!(drain(&mut sub, ordered), [4], "{case}"); + } + } + } + } + #[test] fn arrival_reads_still_deliver_a_lower_group_that_arrives_later() { let routes = Producer::new(); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index f00c325a3f5..08f464c6461 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -5701,6 +5701,36 @@ enum Inner { ), } +/// A group a [`Subscriber`] reserved without handing it out; see +/// [`Subscriber::poll_peek_group`]. It holds the group and the step that moves past it, +/// so taking it never asks the cursor again: a route or bound that moved meanwhile cannot +/// lose it. +#[allow( + clippy::large_enum_variant, + reason = "a plain read, the hot path, passes its group on the stack instead of allocating" +)] +pub(super) enum Reserved { + Plain(Peeked), + Resume(Box), +} + +impl Reserved { + pub(super) fn sequence(&self) -> u64 { + match self { + Self::Plain(peeked) => peeked.group.sequence, + Self::Resume(candidate) => candidate.sequence(), + } + } + + /// Whether it was found in sequence order rather than arrival order. + pub(super) fn ordered(&self) -> bool { + match self { + Self::Plain(peeked) => peeked.arrival.is_none(), + Self::Resume(candidate) => candidate.ordered(), + } + } +} + /// One poll's view of how far this subscription may drift: the clamped budget and the /// live edge to measure a candidate group against. Resolved once, then applied to every /// group that poll considers. @@ -6085,9 +6115,6 @@ struct Cursor { /// `u64::MAX` and no group can follow. /// Used only by that method to skip late arrivals; does not affect `recv_group`. next_sequence: Option, - /// The exact candidate a route-copy selection inspected, including its arrival - /// position. Consuming it must not look up a different group after eviction. - peeked: Option>, /// Exclusive upper sequence bound for `next_group` and `recv_group`. `None` means no /// cap. Set by `set_range`; /// can be raised, lowered, or unset at any time. Groups at or past the cap stay in @@ -6117,10 +6144,12 @@ struct Cursor { misses: Option>, } -struct Peeked { +/// A group a cursor found but has not moved past; see [`Cursor::take`]. +pub(super) struct Peeked { group: group::Consumer, - taken: Taken, - ordered: bool, + /// How taking it moves the cursor: an arrival read past where it came from, or a + /// sequence read past its sequence. + arrival: Option, } /// Where a recv-order candidate came from, which taking it advances past. @@ -6318,7 +6347,6 @@ impl Cursor { index: 0, datagram_index, next_sequence: Some(0), - peeked: None, end_sequence: None, // Uncapped, the servable groups are the window's. servable: published.servable.clone(), @@ -6833,21 +6861,30 @@ impl Cursor { self.misses.as_ref().is_some_and(|misses| misses.is_waiting()) } - fn poll_recv_group( - &mut self, - take: bool, - taken: &Delivered, - waiter: &kio::Waiter, - ) -> Poll>> { - if let Some(peeked) = self.peeked.take().filter(|peeked| take && !peeked.ordered) { - // The reserved group is still the one handed out, but bounded by the ranges in - // force now, which a `Control` may have replaced since the peek. - self.sync(); - self.advance_arrival(peeked.group.sequence, peeked.taken); - self.skipped.remove(peeked.group.sequence); - peeked.group.cache_refresh(); - return Poll::Ready(Ok(Some(self.deliver(peeked.group)))); + /// Find the next group in arrival order, or in sequence order for `ordered`, without + /// moving past it. + fn poll_peek(&mut self, ordered: bool, taken: &Delivered, waiter: &kio::Waiter) -> Poll>> { + match ordered { + true => self.poll_peek_next(taken, waiter), + false => self.poll_peek_recv(taken, waiter), } + } + + /// Move past the group `peeked` found and hand it out: the exact group, even if its + /// producer evicted it since, so eviction cannot advance the reader past a gap. + fn take(&mut self, peeked: Peeked) -> group::Consumer { + let Peeked { group, arrival } = peeked; + let Some(taken) = arrival else { + return self.advance_sequence(group); + }; + self.advance_arrival(group.sequence, taken); + self.skipped.remove(group.sequence); + // Taking is the cache access a peek is not. + group.cache_refresh(); + self.deliver(group) + } + + fn poll_peek_recv(&mut self, taken: &Delivered, waiter: &kio::Waiter) -> Poll>> { self.sync(); if let Some(err) = &self.filter.refused { return Poll::Ready(Err(err.clone())); @@ -6858,17 +6895,11 @@ impl Cursor { let drift = ready!(self.poll_drift(&self.servable, waiter))?; self.reoffer(drift.budget); self.probe(self.floor(), &drift, taken, waiter); - self.poll_recv_with(take, drift, waiter) + self.poll_recv_with(drift, waiter) } - /// [`Self::poll_recv_group`] once synced, judging against `drift`. Without `take`, it - /// peeks: the candidate stays at its cursor position until a taking read. - fn poll_recv_with( - &mut self, - take: bool, - mut drift: Drift, - waiter: &kio::Waiter, - ) -> Poll>> { + /// [`Self::poll_peek_recv`] once synced, judging against `drift`. + fn poll_recv_with(&mut self, mut drift: Drift, waiter: &kio::Waiter) -> Poll>> { loop { // Re-offer what a widening newly covers ahead of the arrival walk, in the // subscription's order: it is the oldest thing still owed. A group the walk has @@ -6880,15 +6911,7 @@ impl Cursor { None => self.next_filled().map(|producer| (producer, Taken::Filled)), }; let (consumer, taken) = match owed { - Some((producer, taken)) => { - let consumer = producer.consume(); - // A re-offer is a delivery: stamp it like a fresh hand-out. Peeking is - // not an access. - if take { - consumer.cache_refresh(); - } - (consumer, taken) - } + Some((producer, taken)) => (producer.consume(), taken), None => { let floor = self.floor(); let polled = self.poll(waiter, |state| state.poll_recv_group(self.index, floor)); @@ -6906,11 +6929,6 @@ impl Cursor { return Poll::Ready(Ok(None)); }; let consumer = producer.consume(); - // Stamp with the track guard released, so delivery never nests the - // group's state lock under the track's. Peeking is not an access. - if take { - consumer.cache_refresh(); - } // The first group to arrive resolves a relative floor, which may land // above it, and a later one may settle where a time end falls. @@ -6940,13 +6958,8 @@ impl Cursor { // Drop a group the drift budget has given up on and keep scanning, so one // poll walks a whole backlog off rather than handing it out group by group. - let stale = ready!(self.poll_stale(&consumer, &drift, waiter))?; - // A peek discards what cannot be delivered, but keeps its candidate at the - // same cursor position until the reader chooses this copy. - if take || stale { + if ready!(self.poll_stale(&consumer, &drift, waiter))? { self.advance_arrival(consumer.sequence, taken); - } - if stale { self.stale.add(consumer.content()); if let Some(drops) = &mut self.drops && let Some(end) = consumer.sequence.checked_add(1) @@ -6960,16 +6973,9 @@ impl Cursor { } continue; } - return Poll::Ready(Ok(Some(if take { - self.skipped.remove(consumer.sequence); - self.deliver(consumer) - } else { - self.peeked = Some(Box::new(Peeked { - group: consumer.clone(), - taken, - ordered: false, - })); - consumer + return Poll::Ready(Ok(Some(Peeked { + group: consumer, + arrival: Some(taken), }))); } } @@ -7026,13 +7032,6 @@ impl Cursor { return; } self.next_sequence = next; - if self - .peeked - .as_ref() - .is_some_and(|peeked| next.is_none_or(|next| peeked.group.sequence < next)) - { - self.peeked = None; - } // A group delivered on another copy is not a stale skip on this one. if !self.seek_pending.is_empty() { self.seek_pending = next.map_or_else(BTreeMap::new, |next| self.seek_pending.split_off(&next)); @@ -7041,17 +7040,7 @@ impl Cursor { /// The lowest servable group past the last one returned, walking off everything the /// drift budget has convicted on the way. - fn poll_next_group( - &mut self, - take: bool, - taken: &Delivered, - waiter: &kio::Waiter, - ) -> Poll>> { - if let Some(peeked) = self.peeked.take().filter(|peeked| take && peeked.ordered) { - // Bounded by the ranges in force now, as on the arrival path. - self.sync(); - return Poll::Ready(Ok(Some(self.advance_sequence(peeked.group)))); - } + fn poll_peek_next(&mut self, taken: &Delivered, waiter: &kio::Waiter) -> Poll>> { let Some(next) = self.next_sequence else { return Poll::Ready(Ok(None)); }; @@ -7063,18 +7052,16 @@ impl Cursor { // One scan for the whole poll, so walking a backlog off stays linear in its size. let drift = ready!(self.poll_drift(&self.servable, waiter))?; self.probe(floor, &drift, taken, waiter); - self.poll_next_with(take, floor, drift, waiter) + self.poll_next_with(floor, drift, waiter) } - /// [`Self::poll_next_group`] once synced, from `floor`, judging against `drift`. - /// Without `take`, it peeks: the candidate is reserved, not delivered. + /// [`Self::poll_peek_next`] once synced, from `floor`, judging against `drift`. fn poll_next_with( &mut self, - take: bool, mut floor: u64, mut drift: Drift, waiter: &kio::Waiter, - ) -> Poll>> { + ) -> Poll>> { let end = self.end_sequence; // Where the gap below the candidate starts: groups stepped over as stale are cached. let mut from = floor; @@ -7146,16 +7133,7 @@ impl Cursor { break group; }; - Poll::Ready(Ok(Some(if take { - self.advance_sequence(group) - } else { - self.peeked = Some(Box::new(Peeked { - group: group.clone(), - taken: Taken::Owed, - ordered: true, - })); - group - }))) + Poll::Ready(Ok(Some(Peeked { group, arrival: None }))) } fn advance_sequence(&mut self, group: group::Consumer) -> group::Consumer { @@ -7175,7 +7153,7 @@ impl Cursor { /// /// A conviction only becomes a real skip once a delivery commits past it. Until then /// the entry waits in [`Self::seek_pending`], where a delivered group removes itself - /// (see [`Self::poll_next_group`]). `committed` must be the sequence watermark (one + /// (see [`Self::poll_next_with`]). `committed` must be the sequence watermark (one /// past the last delivery), which only rises. The seek's floor is NOT that: it /// includes the local floor, which can be lowered again, and a conviction it flushed could /// then be delivered after all. @@ -7410,29 +7388,53 @@ impl Subscriber { /// The next group in arrival order, or in sequence order for `ordered`, skipping any /// already handed out. fn poll_group(&mut self, ordered: bool, waiter: &kio::Waiter) -> Poll>> { - let meter = self.stats.meter(); - let res = match &mut self.inner { + match self.poll_peek_group(ordered, waiter) { + Poll::Ready(Ok(Some(reserved))) => Poll::Ready(self.take(reserved).map(Some)), + res => { + // What the peek skipped for age counts even when nothing is handed out. + self.count_stale(&self.stats.meter()); + match res { + Poll::Ready(Ok(None)) => Poll::Ready(self.ended(false)), + res => res.map(|res| res.map(|_| None)), + } + } + } + } + + /// Reserve the next group not yet handed out, without moving past it: [`Self::take`] + /// hands out that exact group, even if its producer evicted it meanwhile. + pub(super) fn poll_peek_group(&mut self, ordered: bool, waiter: &kio::Waiter) -> Poll>> { + match &mut self.inner { Inner::Plain(cursor) => loop { - let res = match ordered { - true => cursor.poll_next_group(true, &self.delivered, waiter), - false => cursor.poll_recv_group(true, &self.delivered, waiter), + let Some(peeked) = ready!(cursor.poll_peek(ordered, &self.delivered, waiter))? else { + return Poll::Ready(Ok(None)); }; - match res { - Poll::Ready(Ok(Some(group))) if self.delivered.contains(group.sequence) => continue, - res => break res, + if !self.delivered.contains(peeked.group.sequence) { + return Poll::Ready(Ok(Some(Reserved::Plain(peeked)))); } + // A replay advances only this copy's cursor, as an ordinary read would. + cursor.take(peeked); }, - Inner::Resume(resume, _) => resume.poll_group(ordered, &self.delivered, waiter), + Inner::Resume(resume, _) => resume + .poll_peek_group(ordered, &self.delivered, waiter) + .map(|res| res.map(|candidate| candidate.map(|candidate| Reserved::Resume(Box::new(candidate))))), + } + } + + /// Hand out the group a [`Self::poll_peek_group`] on this subscriber reserved. + pub(super) fn take(&mut self, reserved: Reserved) -> Result { + let meter = self.stats.meter(); + let ordered = reserved.ordered(); + let res = match (&mut self.inner, reserved) { + (Inner::Plain(cursor), Reserved::Plain(peeked)) => Ok(cursor.take(peeked)), + (Inner::Resume(resume, _), Reserved::Resume(candidate)) => resume.take(*candidate), + _ => unreachable!("a reservation is taken from the subscriber that made it"), }; - if let Poll::Ready(Ok(Some(group))) = &res { + if let Ok(group) = &res { self.hand_out(group.sequence, ordered); } self.count_stale(&meter); - let res = match res { - Poll::Ready(Ok(None)) => Poll::Ready(self.ended(false)), - res => res, - }; - res.map(|res| res.map(|group| group.map(|group| group.with_meter(meter)))) + res.map(|group| group.with_meter(meter)) } /// The error the track's declared end carries, if it does not end cleanly. @@ -7471,43 +7473,17 @@ impl Subscriber { && fin.checked_sub(1).is_none_or(|last| self.next_in_range(last).is_some()) } - /// Reserve the next group without advancing past it, so the subsequent read - /// takes that exact group even if a producer evicts it meanwhile. - pub(super) fn poll_peek_group(&mut self, ordered: bool, waiter: &kio::Waiter) -> Poll>> { - match &mut self.inner { - Inner::Plain(cursor) => loop { - let group = ready!(match ordered { - true => cursor.poll_next_group(false, &self.delivered, waiter), - false => cursor.poll_recv_group(false, &self.delivered, waiter), - })?; - let Some(group) = group else { - return Poll::Ready(Ok(None)); - }; - if !self.delivered.contains(group.sequence) { - return Poll::Ready(Ok(Some(group.sequence))); - } - // A replay advances only this copy's cursor, as an ordinary read would. - ready!(match ordered { - true => cursor.poll_next_group(true, &self.delivered, waiter), - false => cursor.poll_recv_group(true, &self.delivered, waiter), - })?; - }, - Inner::Resume(resume, _) => resume.poll_peek_group(ordered, &self.delivered, waiter), - } - } - /// Pass groups another route copy handed out, preserving the arrival cursor and bounds. pub(super) fn skip_ordered(&mut self, floor: Option) { match &mut self.inner { Inner::Plain(cursor) => cursor.skip_ordered(floor), - Inner::Resume(resume, _) => { + Inner::Resume(..) => { // Its retention decisions also need the containing reader's past prefix. // This records what is no longer owed, without handing a group out. self.delivered.forget_below(floor.unwrap_or(u64::MAX)); if floor.is_none() { self.delivered.insert(u64::MAX); } - resume.skip_ordered(floor); } } } @@ -7908,7 +7884,6 @@ impl Subscriber { let (start, end) = super::subscription::sequence_bounds(groups); match &mut self.inner { Inner::Plain(cursor) => { - cursor.peeked = None; cursor.min_sequence = start; cursor.end_sequence = end; cursor.update_servable(); @@ -12964,13 +12939,13 @@ mod test { append_at(&mut producer, 0); append_at(&mut producer, 1000); let res = match ordered { - true => cursor.poll_next_with(true, 0, drift, &waiter), - false => cursor.poll_recv_with(true, drift, &waiter), + true => cursor.poll_next_with(0, drift, &waiter), + false => cursor.poll_recv_with(drift, &waiter), }; - let Poll::Ready(Ok(Some(group))) = res else { + let Poll::Ready(Ok(Some(peeked))) = res else { panic!("group 0 is the newest group its range reads (ordered: {ordered})"); }; - assert_eq!(group.sequence, 0); + assert_eq!(peeked.group.sequence, 0); } } @@ -15046,6 +15021,13 @@ mod test { assert_eq!(group.sequence, 5); } + /// The sequence a peek reserves, without taking it. + fn peek(subscriber: &mut Subscriber, ordered: bool) -> Poll> { + subscriber + .poll_peek_group(ordered, &kio::Waiter::noop()) + .map(|res| res.unwrap().map(|reserved| reserved.sequence())) + } + #[test] fn skipping_another_copys_groups_preserves_arrival_bounds_and_the_last_sequence() { let producer = track_producer("test", Info::timed()); @@ -15057,22 +15039,12 @@ mod test { .finish() .unwrap(); } - let waiter = kio::Waiter::noop(); - assert_eq!( - subscriber.poll_peek_group(true, &waiter).map(Result::unwrap), - Poll::Ready(Some(u64::MAX - 1)) - ); + assert_eq!(peek(&mut subscriber, true), Poll::Ready(Some(u64::MAX - 1))); subscriber.skip_ordered(Some(u64::MAX)); - assert_eq!( - subscriber.poll_peek_group(true, &waiter).map(Result::unwrap), - Poll::Ready(Some(u64::MAX)) - ); + assert_eq!(peek(&mut subscriber, true), Poll::Ready(Some(u64::MAX))); subscriber.skip_ordered(None); subscriber.skip_ordered(Some(0)); - assert_eq!( - subscriber.poll_peek_group(true, &waiter).map(Result::unwrap), - Poll::Ready(None) - ); + assert_eq!(peek(&mut subscriber, true), Poll::Ready(None)); assert_eq!(subscriber.delivered.runs(), 0, "skips are not deliveries"); assert_eq!(subscriber.take_stale().groups, 0, "duplicates are not stale skips"); let Inner::Plain(cursor) = &subscriber.inner else { @@ -15081,10 +15053,7 @@ mod test { assert_eq!(cursor.min_sequence, 0); assert_eq!(cursor.end_sequence, None); assert_eq!(cursor.index, 0); - assert_eq!( - subscriber.poll_peek_group(false, &waiter).map(Result::unwrap), - Poll::Ready(Some(u64::MAX - 1)) - ); + assert_eq!(peek(&mut subscriber, false), Poll::Ready(Some(u64::MAX - 1))); } #[test] @@ -15094,22 +15063,19 @@ mod test { append_at(&mut producer, second * 1000); } let mut subscriber = producer.subscribe(None); - let waiter = kio::Waiter::noop(); - assert_eq!( - subscriber.poll_peek_group(true, &waiter).map(Result::unwrap), - Poll::Ready(Some(3)) - ); - subscriber.skip_ordered(Some(2)); - let Poll::Ready(Ok(Some(group))) = subscriber.poll_next_group(&waiter) else { - panic!("the reserved group is still delivered"); + let Poll::Ready(Ok(Some(reserved))) = subscriber.poll_peek_group(true, &kio::Waiter::noop()) else { + panic!("group 3 is servable"); }; - assert_eq!(group.sequence, 3); + assert_eq!(reserved.sequence(), 3); + subscriber.skip_ordered(Some(2)); + assert_eq!(subscriber.take(reserved).unwrap().sequence, 3); assert_eq!(subscriber.take_stale().groups, 1, "only group 2 was skipped for age"); assert_eq!(subscriber.take_stale().groups, 0, "counted once"); } + /// A peek moves no cursor, so the bounds in force at the next read decide it. #[test] - fn changed_bounds_invalidate_a_peeked_group() { + fn changed_bounds_after_a_peek_decide_the_next_read() { for ordered in [false, true] { for raise_floor in [false, true] { let producer = track_producer("test", Info::timed()); @@ -15122,10 +15088,7 @@ mod test { .unwrap(); } let waiter = kio::Waiter::noop(); - assert_eq!( - subscriber.poll_peek_group(ordered, &waiter).map(Result::unwrap), - Poll::Ready(Some(1)) - ); + assert_eq!(peek(&mut subscriber, ordered), Poll::Ready(Some(1))); if raise_floor { subscriber.set_range(2..); let Poll::Ready(Ok(Some(group))) = subscriber.poll_group(ordered, &waiter) else { @@ -15157,17 +15120,13 @@ mod test { .finish() .unwrap(); } - let waiter = kio::Waiter::noop(); - assert_eq!( - subscriber.poll_peek_group(ordered, &waiter).map(Result::unwrap), - Poll::Ready(Some(1)) - ); - evict(&producer, 1); - let Poll::Ready(Ok(Some(group))) = subscriber.poll_group(ordered, &waiter) else { - panic!("the exact reserved group must be returned"); + let Poll::Ready(Ok(Some(reserved))) = subscriber.poll_peek_group(ordered, &kio::Waiter::noop()) else { + panic!("group 1 is servable"); }; + evict(&producer, 1); assert_eq!( - group.sequence, 1, + subscriber.take(reserved).unwrap().sequence, + 1, "ordered={ordered}: eviction must not substitute group 3" ); } @@ -15190,10 +15149,7 @@ mod test { assert!(subscriber.poll_group(false, &waiter).is_pending()); subscriber.set_range(..); for _ in 0..2 { - assert_eq!( - subscriber.poll_peek_group(false, &waiter).map(Result::unwrap), - Poll::Ready(Some(1)) - ); + assert_eq!(peek(&mut subscriber, false), Poll::Ready(Some(1))); } let mut read = Vec::new(); while let Poll::Ready(Ok(Some(group))) = subscriber.poll_group(false, &waiter) { @@ -15217,10 +15173,7 @@ mod test { let waiter = kio::Waiter::noop(); let first = if ordered { 1 } else { 2 }; for _ in 0..2 { - assert_eq!( - subscriber.poll_peek_group(ordered, &waiter).map(Result::unwrap), - Poll::Ready(Some(first)) - ); + assert_eq!(peek(&mut subscriber, ordered), Poll::Ready(Some(first))); } let Poll::Ready(Ok(Some(group))) = subscriber.poll_group(ordered, &waiter) else { panic!("the peeked group is still available"); @@ -15230,10 +15183,7 @@ mod test { subscriber.set_range(..0); assert!(subscriber.poll_peek_group(ordered, &waiter).is_pending()); subscriber.set_range(..); - assert_eq!( - subscriber.poll_peek_group(ordered, &waiter).map(Result::unwrap), - Poll::Ready(Some(second)) - ); + assert_eq!(peek(&mut subscriber, ordered), Poll::Ready(Some(second))); } }