diff --git a/quest/m1/release-perf/README.md b/quest/m1/release-perf/README.md index a35788fa4bb..72f880f2a5c 100644 --- a/quest/m1/release-perf/README.md +++ b/quest/m1/release-perf/README.md @@ -61,7 +61,7 @@ usually pays for a feature. | `track_gap_flush/64` | 1.5 us | 6.2 us | 4.1x | same | | `stats/session-report/1000x10` | 11.0 us | 86.8 us | 7.9x | [Stats report](/quest/m1/release-perf/stats-report.md) | | `track_gc/idle/16384` | 11.6 ns | 73.7 ns | 6.3x | [Idle GC](/quest/m1/release-perf/gc-idle.md) | -| `origin/resume_overlap/sequence/1r_64g_32c` | 93 us | 426 us | 4.6x | [Resume duplicates](/quest/m1/release-perf/resume-duplicates.md) | +| `origin/resume_overlap/sequence/1r_64g_32c` | 93 us | 426 us | 4.6x | fixed in [moq-dev/moq#6292](https://github.com/moq-dev/moq/pull/6292) | | `session_delivery_group_cost/moq-transport-22/.../viewers=64/.../fetch` | 1.19 ms | 3.86 ms | 3.2x | accepted (above) | | `group_write_frames/single/8192` | 424 us | 486 us | 1.17x | [Re-check](/quest/m1/release-perf/recheck.md) | @@ -94,7 +94,6 @@ one. - [FETCH fan-out](/quest/m1/release-perf/fetch-fanout.md) - a relay serves later viewers' lite FETCHes of a group from its cache, so per-viewer cost falls with viewers again - [Sparse gaps](/quest/m1/release-perf/sparse-gaps.md) - an arrival and a gap flush cost O(log gaps), not O(gaps) -- [Resume duplicates](/quest/m1/release-perf/resume-duplicates.md) - an ordered resume skips copies of an already-delivered group without consuming them - [Idle GC](/quest/m1/release-perf/gc-idle.md) - an idle track GC skips the wall-clock pass when nothing is due - [Stats report](/quest/m1/release-perf/stats-report.md) - the stats drain locks each row once and its bench counts the wire walk; per-label bytes stay, at an accepted O(live sessions) - [Re-check](/quest/m1/release-perf/recheck.md) - the screen-only moves are re-measured with instruction counts, and the ones that hold are fixed or accepted diff --git a/quest/m1/release-perf/resume-duplicates.md b/quest/m1/release-perf/resume-duplicates.md deleted file mode 100644 index 86b79a9aca4..00000000000 --- a/quest/m1/release-perf/resume-duplicates.md +++ /dev/null @@ -1,33 +0,0 @@ -# [M] Resume duplicates - -## Goal - -An ordered resume reader skips a route copy's groups that another copy -already delivered, without consuming them. `origin/resume_overlap/sequence` -costs about one cheap skip per duplicate again. - -## Plan - -Found by reading code on 2026-10-21 (4.6x at `1r_64g_32c`, confirmed). - -- Release: each segment did a non-consuming `poll_seek_group(floor, cap)`, so - a duplicate below the floor cost nothing. -- Main: each copy has its own `track::Subscriber` (from - [#4741](https://github.com/moq-dev/moq/pull/4741)), and the `Delivered` - dedupe comes from [#5446](https://github.com/moq-dev/moq/pull/5446). For - each group handed out, `resume::Reader::poll_lowest` peeks all 32 copies, - and [#6019](https://github.com/moq-dev/moq/pull/6019) added peeking every - copy (its own PR measured about 1.3x of the cost). For each copy whose head - was already delivered, it then fully consumes the head with - `sub.poll_next_group`. That runs the meter, the inner `hand_out`, - `forget_past_budget`, `count_stale`, and builds a group consumer, only to - discard it. - -Fix: in ordered mode, move a copy's cursor up to the floor with a seek that -builds no consumer, meter entry, or `hand_out`, as release did per segment. -Cache each copy's peek across polls and re-peek only the copy that delivered. -#6019 also lists skipping peek-then-take when only one live copy remains; do -that here if it falls out of the same change. - -Before fixing, check whether `origin/arrival/1r_64g_32c` regressed by the -same factor. If it didn't, that confirms the lost seek is the cause. diff --git a/rs/moq-net/benches/origin.rs b/rs/moq-net/benches/origin.rs index e27d5f695a0..7994148914c 100644 --- a/rs/moq-net/benches/origin.rs +++ b/rs/moq-net/benches/origin.rs @@ -1337,6 +1337,20 @@ struct ResumeOverlap { readers: Vec, } +impl ResumeOverlap { + #[inline(never)] + fn drain(&mut self, retained: usize) { + let waiter = kio::Waiter::noop(); + for reader in &mut self.readers { + let mut delivered = 0; + while let Poll::Ready(Ok(Some(_))) = reader.poll_group(&waiter) { + delivered += 1; + } + assert_eq!(delivered, retained, "every group once"); + } + } +} + enum ResumeReader { Arrival(track::Subscriber), Sequence(track::Ordered), @@ -1351,7 +1365,20 @@ impl ResumeReader { } } -async fn resume_overlap(readers: usize, retained: usize, routes: usize, ordered: bool) -> ResumeOverlap { +struct ResumeShape { + readers: usize, + retained: usize, + routes: usize, + waiting: bool, +} + +async fn resume_overlap(shape: ResumeShape, ordered: bool) -> ResumeOverlap { + let ResumeShape { + readers, + retained, + routes, + waiting, + } = shape; let (producer, driver) = origin::Producer::new(origin::Config::default()); tokio::spawn(moq_net::time::run(driver)); let epoch = Epoch::mint(); @@ -1391,8 +1418,14 @@ async fn resume_overlap(readers: usize, retained: usize, routes: usize, ordered: subscribers.push(subscriber); } let waiter = kio::Waiter::noop(); - for _ in 1..routes { - generations.push(generation(&mut (0..retained).rev())); + for route in 1..routes + usize::from(waiting) { + generations.push(if waiting && route == routes { + // The serving copy has answered but not produced a group. It has not yet + // bounded the overlapping copies, which must still skip each other's replay. + generation(&mut std::iter::empty()) + } else { + generation(&mut (0..retained).rev()) + }); let (_, latest) = generations.last().unwrap(); moq_clock::timeout(COPY_WAIT, support::wanted(latest.demand())) .await @@ -1429,8 +1462,16 @@ async fn resume_overlap(readers: usize, retained: usize, routes: usize, ordered: fn bench_resume_overlap(c: &mut Criterion) { let mut group = c.benchmark_group("origin/resume_overlap"); group.sample_size(10); - for (order, ordered) in [("arrival", false), ("sequence", true)] { + for (order, ordered, waiting) in [ + ("arrival", false, false), + ("sequence", true, false), + ("arrival_waiting", false, true), + ("sequence_waiting", true, true), + ] { for (readers, retained, routes) in RESUME_OVERLAP { + if waiting && !(readers == 1 && retained == 64 && routes >= 2) { + continue; + } group.throughput(Throughput::Elements((readers * retained) as u64)); group.bench_function( BenchmarkId::new(order, format!("{readers}r_{retained}g_{routes}c")), @@ -1439,18 +1480,19 @@ fn bench_resume_overlap(c: &mut Criterion) { .enable_all() .build() .unwrap(); - let waiter = kio::Waiter::noop(); b.iter_batched_ref( - || runtime.block_on(resume_overlap(readers, retained, routes, ordered)), - |rig| { - for reader in &mut rig.readers { - let mut delivered = 0; - while let Poll::Ready(Ok(Some(_))) = reader.poll_group(&waiter) { - delivered += 1; - } - assert_eq!(delivered, retained, "every group once"); - } + || { + runtime.block_on(resume_overlap( + ResumeShape { + readers, + retained, + routes, + waiting, + }, + ordered, + )) }, + |rig| rig.drain(retained), BatchSize::PerIteration, ); }, diff --git a/rs/moq-net/src/model/delivered.rs b/rs/moq-net/src/model/delivered.rs index 6ee61a91267..b21c00ddd34 100644 --- a/rs/moq-net/src/model/delivered.rs +++ b/rs/moq-net/src/model/delivered.rs @@ -22,6 +22,11 @@ impl Delivered { .is_some_and(|(_, last)| sequence <= *last) } + /// The first sequence above the handed-out prefix, or none once every sequence is past. + pub fn floor(&self) -> Option { + self.runs.get(&0).map_or(Some(0), |last| last.checked_add(1)) + } + /// Whether every sequence in `groups` was handed out, or is below the floor. pub fn covers(&self, groups: std::ops::Range) -> bool { groups.is_empty() @@ -104,6 +109,7 @@ mod test { assert_eq!(runs(&delivered), [(4, 8)]); assert!(delivered.contains(4) && delivered.contains(8)); assert!(!delivered.contains(3) && !delivered.contains(9)); + assert_eq!(delivered.floor(), Some(0), "a run above a gap is not the prefix"); assert!(delivered.covers(4..9) && delivered.covers(9..9)); assert!(!delivered.covers(3..5) && !delivered.covers(8..10)); } @@ -117,6 +123,7 @@ mod test { assert_eq!(runs(&delivered), [(0, 0), (u64::MAX, u64::MAX)]); delivered.forget_below(u64::MAX); assert_eq!(runs(&delivered), [(0, u64::MAX)]); + assert_eq!(delivered.floor(), None); } #[test] @@ -134,6 +141,7 @@ mod test { assert_eq!(runs(&delivered), [(0, 6), (9, 9), (12, 12)]); assert_eq!(delivered.above_gap(), Some(9)); assert!(delivered.contains(4), "below the floor counts as handed out"); + assert_eq!(delivered.floor(), Some(7)); // A floor inside a gap keeps the run above it. delivered.forget_below(11); diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 04b5086e3f4..a62d857a1aa 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -576,6 +576,18 @@ impl Subscriber { } } + /// Invalidate a reservation below the containing reader's past prefix. + pub(super) fn skip_ordered(&mut self, floor: Option) { + let mut reader = self.reader(); + if reader + .peeked + .as_ref() + .is_some_and(|peeked| floor.is_none_or(|floor| peeked.sequence < floor)) + { + reader.peeked = None; + } + } + /// Poll for the next datagram in arrival order. `delivered` decides which replaced /// copies still have a group to hand out, once groups were asked for. pub(super) fn poll_recv_datagram( @@ -1112,12 +1124,18 @@ impl Reader { self.groups_read = true; let groups = self.groups; let budget = self.mirrored.max_delay; + let floor = if ordered { delivered.floor() } else { Some(0) }; let mut lowest: Option = None; for index in 0..self.copies.len() { let copy = &mut self.copies[index]; // What a replaced copy holds is past the reader's budget once a newer group // superseded it a full budget ago. - if copy.done.is_some() || copy.expired(budget, waiter) { + // A fully handed-out bounded copy still backs groups being read, but cannot + // supply another one. Keep its recovery lease without polling its empty range. + if copy.done.is_some() + || copy.end().is_some_and(|end| delivered.covers(groups.0..end)) + || copy.expired(budget, waiter) + { continue; } loop { @@ -1130,6 +1148,9 @@ impl Reader { } Poll::Ready(Ok(sub)) => sub, }; + if ordered { + sub.skip_ordered(floor); + } let res = sub.poll_peek_group(ordered, waiter); match res { Poll::Pending => break, @@ -1138,10 +1159,11 @@ impl Reader { let Sub::Ready(sub) = &mut self.copies[index].sub else { unreachable!() }; - let _ = match ordered { - true => sub.poll_next_group(waiter), - false => sub.poll_recv_group(waiter), - }; + if ordered { + sub.skip_ordered(sequence.checked_add(1)); + } else { + let _ = sub.poll_recv_group(waiter); + } continue; } if lowest.as_ref().is_none_or(|lowest| sequence <= lowest.sequence) { @@ -2495,6 +2517,96 @@ mod test { } } + #[test] + fn a_fresh_nested_reader_ends_after_its_outer_reader_already_passed_its_groups() { + let routes = Producer::new(); + let outer = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&outer, HOUR); + for sequence in 0..4 { + write(&a, sequence); + } + assert_eq!(drain(&mut sub, true), [0, 1, 2, 3]); + + let nested_routes = Producer::new(); + let nested = logical(&nested_routes); + let b = copy(); + for sequence in 0..4 { + write(&b, sequence); + } + nested_routes.serve(b.consume()); + routes.serve(nested.consume()); + let waiter = kio::Waiter::noop(); + assert!(sub.poll_next_group(&waiter).is_pending()); + + // The nested reader has handed nothing out itself. Its containing reader's + // past prefix must also release the replaced copy when its serving copy ends. + let c = copy(); + for sequence in 0..4 { + write(&c, sequence); + } + c.finish().unwrap(); + nested_routes.serve(c.consume()); + assert!(matches!(sub.poll_next_group(&waiter), Poll::Ready(Ok(None)))); + } + + #[test] + fn ordered_replay_skips_unbounded_copies_while_a_nested_route_waits() { + for nested in [false, true] { + let routes = Producer::new(); + let inner = logical(&routes); + let outer_routes = Producer::new(); + let outer = logical(&outer_routes); + outer_routes.serve(inner.consume()); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(if nested { &outer } else { &inner }, HOUR); + sub.set_groups(..0); + for sequence in 0..4 { + write(&a, sequence); + } + assert!( + drain(&mut sub, true).is_empty(), + "subscribe the nested reader before switching" + ); + let b = copy(); + for sequence in 0..4 { + write(&b, sequence); + } + routes.serve(b.consume()); + assert!(drain(&mut sub, true).is_empty()); + let c = track::Request::new(Arc::new(broadcast::Info::default()), "video"); + routes.serve(c.consume()); + assert!(drain(&mut sub, true).is_empty()); + sub.set_groups(..); + assert_eq!(drain(&mut sub, true), [0, 1, 2, 3], "nested={nested}"); + { + let readers = routes.readers.lock().unwrap(); + let reader = readers.iter().find_map(Weak::upgrade).unwrap(); + let reader = reader.lock().unwrap(); + let Sub::Ready(older) = &reader.copies[0].sub else { + panic!("older copy ready") + }; + assert_eq!( + older.delivered_runs(), + 0, + "duplicates were never handed out on the older copy: nested={nested}" + ); + } + let c = c.accept(crate::track::Info::timed()); + for sequence in 0..5 { + write(&c, sequence); + } + assert_eq!(drain(&mut sub, true), [4], "a newly ready copy inherits the floor"); + c.finish().unwrap(); + assert!( + matches!(sub.poll_next_group(&kio::Waiter::noop()), Poll::Ready(Ok(None))), + "nested end propagates past covered copies" + ); + } + } + /// A replaced copy is bounded only once the new copy delivers its first group: a group /// buffered on it must be read before a later group on the serving copy. #[test] diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 5a0f04c654d..3b75eb5b41f 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -6105,6 +6105,26 @@ impl Cursor { } } + /// Pass groups another copy already handed out, without delivering or metering them. + fn skip_ordered(&mut self, floor: Option) { + let next = self.next_sequence.zip(floor).map(|(next, floor)| next.max(floor)); + if next == self.next_sequence { + return; + } + self.next_sequence = next; + if self + .peeked + .as_ref() + .is_some_and(|peeked| next.is_none_or(|next| peeked.group.sequence < next)) + { + self.peeked = None; + } + // A group delivered on another copy is not a stale skip on this one. + if !self.seek_pending.is_empty() { + self.seek_pending = next.map_or_else(BTreeMap::new, |next| self.seek_pending.split_off(&next)); + } + } + /// The lowest servable group past the last one returned, walking off everything the /// drift budget has convicted on the way. fn poll_next_group(&mut self, take: bool, waiter: &kio::Waiter) -> Poll>> { @@ -6455,6 +6475,22 @@ impl Subscriber { } } + /// Pass groups another route copy handed out, preserving the arrival cursor and bounds. + pub(super) fn skip_ordered(&mut self, floor: Option) { + match &mut self.inner { + Inner::Plain(cursor) => cursor.skip_ordered(floor), + Inner::Resume(resume, _) => { + // Its retention decisions also need the containing reader's past prefix. + // This records what is no longer owed, without handing a group out. + self.delivered.forget_below(floor.unwrap_or(u64::MAX)); + if floor.is_none() { + self.delivered.insert(u64::MAX); + } + resume.skip_ordered(floor); + } + } + } + /// Record `sequence` as handed out. An ordered read never goes back, and any reader /// stops expecting what is past its budget, so only the gaps within it cost memory. fn hand_out(&mut self, sequence: u64, ordered: bool) { @@ -12982,6 +13018,68 @@ mod test { assert_eq!(group.sequence, 5); } + #[test] + fn skipping_another_copys_groups_preserves_arrival_bounds_and_the_last_sequence() { + let producer = track_producer("test", Info::timed()); + let mut subscriber = producer.subscribe(replay()); + for sequence in [u64::MAX - 1, u64::MAX] { + producer + .create_group(group::Info { sequence }) + .unwrap() + .finish() + .unwrap(); + } + let waiter = kio::Waiter::noop(); + assert_eq!( + subscriber.poll_peek_group(true, &waiter).map(Result::unwrap), + Poll::Ready(Some(u64::MAX - 1)) + ); + subscriber.skip_ordered(Some(u64::MAX)); + assert_eq!( + subscriber.poll_peek_group(true, &waiter).map(Result::unwrap), + Poll::Ready(Some(u64::MAX)) + ); + subscriber.skip_ordered(None); + subscriber.skip_ordered(Some(0)); + assert_eq!( + subscriber.poll_peek_group(true, &waiter).map(Result::unwrap), + Poll::Ready(None) + ); + assert_eq!(subscriber.delivered.runs(), 0, "skips are not deliveries"); + assert_eq!(subscriber.take_stale().groups, 0, "duplicates are not stale skips"); + let Inner::Plain(cursor) = &subscriber.inner else { + unreachable!() + }; + assert_eq!(cursor.min_sequence, 0); + assert_eq!(cursor.end_sequence, None); + assert_eq!(cursor.index, 0); + assert_eq!( + subscriber.poll_peek_group(false, &waiter).map(Result::unwrap), + Poll::Ready(Some(u64::MAX - 1)) + ); + } + + #[test] + fn an_ordered_skip_counts_only_convictions_not_delivered_elsewhere() { + let mut producer = track_producer("test", Info::timed()); + for second in 0..4 { + append_at(&mut producer, second * 1000); + } + let mut subscriber = producer.subscribe(None); + let waiter = kio::Waiter::noop(); + assert_eq!( + subscriber.poll_peek_group(true, &waiter).map(Result::unwrap), + Poll::Ready(Some(3)) + ); + subscriber.skip_ordered(Some(2)); + let Poll::Ready(Ok(Some(group))) = subscriber.poll_next_group(&waiter) else { + panic!("the reserved group is still delivered"); + }; + assert_eq!(group.sequence, 3); + assert_eq!(subscriber.take_stale().groups, 1, "only group 2 was skipped for age"); + assert_eq!(subscriber.take_stale().groups, 0, "counted once"); + } + #[test] fn changed_bounds_invalidate_a_peeked_group() { for ordered in [false, true] {