diff --git a/quest/m2/performance-survey/README.md b/quest/m2/performance-survey/README.md index 7c5faa7e3d6..08475164a41 100644 --- a/quest/m2/performance-survey/README.md +++ b/quest/m2/performance-survey/README.md @@ -189,7 +189,6 @@ No duplicate quest or workflow edit belongs here. - [Cached JS group joins](/quest/m2/performance-survey/js-cached-groups.md) - repeated sorting while replaying retained groups - [PCM batch processing](/quest/m2/performance-survey/audio-batches.md) - repeated tail movement during encoding and resampling - [Fragmented MKV import](/quest/m2/performance-survey/mkv-fragments.md) - accumulated-buffer copies and parser replay -- [QUIC scheduling churn](/quest/m2/performance-survey/quic-schedule-churn.md) - priority and retirement work across unrelated groups - [HLS request fan-out](/quest/m2/performance-survey/hls-serving.md) - repeated fetch, remux and playlist construction per viewer - [Buffered browser groups](/quest/m2/performance-survey/hang-buffered.md) - per-frame scans and bulk skip work - [Idle browser stats](/quest/m2/performance-survey/browser-idle-stats.md) - local transport polling without active publishing demand diff --git a/quest/m2/performance-survey/quic-schedule-churn.md b/quest/m2/performance-survey/quic-schedule-churn.md deleted file mode 100644 index 2f05c9d4b4e..00000000000 --- a/quest/m2/performance-survey/quic-schedule-churn.md +++ /dev/null @@ -1,30 +0,0 @@ -# [M] QUIC priority changes touch only relevant scheduling state - -## Goal - -Changing or retiring queued scheduling groups does not impose unnecessary -work on unrelated groups sharing a connection. - -## Plan - -[The QUIC scheduler](/rs/moq-quic/src/connection/streams/schedule.rs) -linearly searches round-robin VecDeques in `unring_group` and retirement. -Priority changes call that path. On shared relay sessions, -repeated updates or unfavorable retirement order can multiply changed groups -by unrelated queued groups. Existing dispatch/stream-churn benches do not -isolate scheduling-group reprioritization and retirement. - -Sweep queued groups × changed/retired groups, and domains × retired domains. -Measure front/middle/tail and mixed-order operations, priority-update CPU, -allocations and packet-dispatch tail latency during churn. Confirm through -publisher × subscriber traffic on a shared session. Preserve byte fairness, -priority ordering and eventual progress rather than changing scheduler policy. - -Benchmark the priority semantics from -[moq-dev/moq#6142](https://github.com/moq-dev/moq/pull/6142). No new -priority contract is selected by this performance quest. - -Follow the [survey's benchmark contract](/quest/m2/performance-survey/README.md): -measure the baseline before optimizing, retain paired evidence, verify equivalent -work, schedule coverage nightly, and accept a measured no-win. Expected public -API and wire impact: none; report the actual impact in the implementation PR. diff --git a/rs/moq-quic/benches/streams.rs b/rs/moq-quic/benches/streams.rs index d5fef4f2f68..37f36bb8247 100644 --- a/rs/moq-quic/benches/streams.rs +++ b/rs/moq-quic/benches/streams.rs @@ -4,7 +4,8 @@ //! finish them, then exchange packets until the server has read them and the path is idle. //! The zero-credit case isolates blocked writes and resets across live-writer counts, with //! connection setup and dropping the remaining state outside the CPU measurement. The -//! reprioritize case moves streams that stay queued, never sending them. +//! reprioritize case moves streams that stay queued, never sending them. The group churn case +//! moves and cancels send groups, one per subscription, while the rest have data queued. use bytes::Bytes; use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; @@ -65,6 +66,57 @@ fn bench_schedule(c: &mut Criterion) { ); } } + + // Moving or retiring some queued groups, as subscription updates and cancels do, while the + // rest stay queued. Groups are picked from the front of the round robin, its tail, or spread + // across it, so a cost that grows with the untouched groups shows as a slope. + let queued_groups = |groups: usize| { + let mut schedule = Schedule::::new(); + let ids: Vec<_> = (0..groups).map(|_| schedule.open_group(0)).collect(); + let mut slots: Vec<_> = (0..groups).map(|_| Slot::default()).collect(); + for (id, slot) in slots.iter_mut().enumerate() { + schedule.join(id, slot, Some(ids[id]), 0); + schedule.push(id, slot, 0); + } + (schedule, ids, slots) + }; + for (queued, changed) in [(16, 16), (256, 16), (4096, 16), (4096, 256)] { + for order in ["front", "tail", "mixed"] { + let picked: Vec = match order { + "front" => (0..changed).collect(), + "tail" => (queued - changed..queued).rev().collect(), + // An odd stride visits distinct groups of a power-of-two count. + _ => (0..changed).map(|i| i * 2_654_435_761 % queued).collect(), + }; + let id = format!("queued_{queued}_changed_{changed}_{order}"); + group.throughput(Throughput::Elements(changed as u64)); + group.bench_function(BenchmarkId::new("group_priority", &id), |b| { + b.iter_batched( + || queued_groups(queued), + |(mut schedule, ids, slots)| { + for &index in &picked { + schedule.set_group_priority(ids[index], 1); + } + (schedule, slots) + }, + BatchSize::LargeInput, + ) + }); + group.bench_function(BenchmarkId::new("group_retire", &id), |b| { + b.iter_batched( + || queued_groups(queued), + |(mut schedule, ids, mut slots)| { + for &index in &picked { + schedule.close_group(ids[index]); + schedule.leave(&mut slots[index]); + } + (schedule, slots) + }, + BatchSize::LargeInput, + ) + }); + } + } group.finish(); } @@ -143,6 +195,69 @@ fn bench_reprioritize(c: &mut Criterion) { group.finish(); } +/// Subscription churn on a shared connection: each round writes one small stream in each of +/// `subs` send groups, then, with all of it queued, moves `churn` groups spread across the +/// round robin to another priority and back and cancels `churn` others (reset the stream, +/// close the group, open its replacement), and drives until the server has seen every stream. +fn bench_group_churn(c: &mut Criterion) { + use moq_quic::send; + const BYTES: usize = 256; + let data = payload(BYTES); + let mut group = c.benchmark_group("group_churn"); + for (subs, churn) in [(16_usize, 4), (256, 16), (4096, 16), (4096, 256)] { + // The moved and cancelled groups stay distinct, below. + assert!(2 * churn <= subs && subs.is_power_of_two()); + let mut pair = Pair::new(Setup::default()); + let mut groups: Vec<_> = (0..subs).map(|_| pair.client().send_groups().open(0)).collect(); + let mut round = 0; + group.throughput(Throughput::Elements(subs as u64)); + group.bench_function( + BenchmarkId::new(format!("subs_{subs}"), format!("churn_{churn}")), + |b| { + b.iter(|| { + round += 1; + let finished = pair.server.received.finished; + let reset = pair.server.received.reset; + let conn = pair.client(); + let ids: Vec<_> = groups + .iter() + .map(|group| { + let options = send::Options { + group: Some(group), + priority: 0, + }; + let id = conn.streams().open(Dir::Uni, options).expect("stream limit"); + let mut stream = conn.send_stream(id); + assert_eq!(stream.write(&mut data.clone()).unwrap(), BYTES); + stream.finish().unwrap(); + id + }) + .collect(); + // An odd stride permutes a power-of-two count, spreading the touched groups over + // the round robin and shifting them each round. + let spread = |i: usize| (round + i * 2_654_435_761) % subs; + for index in (0..churn).map(spread) { + conn.send_groups().set_priority(&groups[index], 1); + conn.send_groups().set_priority(&groups[index], 0); + } + for index in (churn..2 * churn).map(spread) { + conn.send_stream(ids[index]).reset(0u32.into()).unwrap(); + let fresh = conn.send_groups().open(0); + conn.send_groups().close(std::mem::replace(&mut groups[index], fresh)); + } + let target = (finished + (subs - churn) as u64, reset + churn as u64); + pair.drive_until(|pair| { + let received = &pair.server.received; + received.finished >= target.0 && received.reset >= target.1 + }); + pair.settle(); + }) + }, + ); + } + group.finish(); +} + /// Writes `total` bytes to one stream in `size`-byte writes, exchanging packets whenever the /// client's send window is full, and drives until the server has read them all. fn send_windowed(pair: &mut Pair, total: usize, size: usize) { @@ -296,6 +411,6 @@ fn bench_probe(c: &mut Criterion) { criterion_group! { name = benches; config = common::config(); - targets = bench_schedule, bench_priority, bench_reprioritize, bench_write_size, bench_connection_blocked, bench_probe + targets = bench_schedule, bench_priority, bench_reprioritize, bench_group_churn, bench_write_size, bench_connection_blocked, bench_probe } criterion_main!(benches); diff --git a/rs/moq-quic/src/connection/streams/schedule.rs b/rs/moq-quic/src/connection/streams/schedule.rs index d92f6e8559f..3684ba9e6c7 100644 --- a/rs/moq-quic/src/connection/streams/schedule.rs +++ b/rs/moq-quic/src/connection/streams/schedule.rs @@ -20,7 +20,7 @@ use std::{ cmp::Reverse, - collections::{BTreeMap, BTreeSet, VecDeque}, + collections::{BTreeMap, BTreeSet, btree_map}, }; use slab::Slab; @@ -147,10 +147,10 @@ struct Group { /// Whether the group has queued items, or is being served a stream. It stays set between /// [`Schedule::pop`] and [`Schedule::sent`], so requeueing the stream is not a new turn. active: bool, - /// Whether an entry for this group sits in its level's round robin. Usually equal to - /// having queued items, but a group that runs dry away from the front of the round - /// robin keeps its entry until the round robin reaches it. - ringed: bool, + /// The group's neighbors in its level's round robin, if it is in one. Usually present + /// exactly when the group has queued items, but a group that runs dry away from the front + /// of the round robin keeps its place until the round robin reaches it. + ring: Option, /// Streams that name this group, queued or not. members: usize, /// Whether the owner released the group, which is freed once no stream names it and its @@ -158,6 +158,14 @@ struct Group { closed: bool, } +/// A group's neighbors in its level's round robin: a circular list threaded through the +/// groups, so a group leaves it from anywhere without a search. +#[derive(Debug, Clone, Copy)] +struct Ring { + prev: usize, + next: usize, +} + /// A queue holding datagrams, keyed for eviction: whether it is the ungrouped queue, its /// group's priority, its oldest datagram, and the group. type Victim = (bool, i64, Reverse, Option); @@ -167,8 +175,9 @@ type Victim = (bool, i64, Reverse, Option); pub(super) struct Schedule { ungrouped: Queue, groups: Slab>, - /// Group round robins by priority, with only the highest backlogged level served. - levels: BTreeMap>, + /// The group heading each priority's round robin, with only the highest backlogged level + /// served. A level goes with its last group. + levels: BTreeMap, /// Groups with queued items. backlogged: usize, /// Decreases with every push, so a requeued stream goes behind its equal-priority peers. @@ -236,7 +245,7 @@ impl Schedule { queue: Queue::new(), deficit: 0, active: false, - ringed: false, + ring: None, members: 0, closed: false, }) @@ -259,15 +268,14 @@ impl Schedule { } self.unindex(Some(group)); + let ringed = self.groups[group].ring.is_some(); + if ringed { + self.unring(group); + } let state = &mut self.groups[group]; - let old = std::mem::replace(&mut state.priority, priority); - if state.ringed { - self.unring_group(group, old); - let state = &mut self.groups[group]; - state.ringed = !state.queue.is_empty(); - if state.ringed { - self.levels.entry(priority).or_default().push_back(group); - } + state.priority = priority; + if ringed && !state.queue.is_empty() { + self.ring(group); } self.index(Some(group)); } @@ -478,19 +486,15 @@ impl Schedule { loop { let mut level = self.levels.last_entry()?; - let Some(&group) = level.get().front() else { - level.remove(); - continue; - }; + let group = *level.get(); let group_state = &mut self.groups[group]; if !group_state.active { - level.get_mut().pop_front(); - group_state.ringed = false; + self.unring(group); continue; } if group_state.deficit <= 0 { group_state.deficit += QUANTUM; - level.get_mut().rotate_left(1); + *level.get_mut() = group_state.ring.expect("a level's head is ringed").next; continue; } @@ -530,7 +534,7 @@ impl Schedule { self.unindex(group); let (datagram, frame) = self.queue(group).take_datagram(recency); self.datagram_bytes -= frame; - if let std::collections::btree_map::Entry::Occupied(mut count) = self.frames.entry(frame) { + if let btree_map::Entry::Occupied(mut count) = self.frames.entry(frame) { *count.get_mut() -= 1; if *count.get() == 0 { count.remove(); @@ -585,9 +589,8 @@ impl Schedule { // Debt carries over, but unused credit does not, so going idle saves nothing up. state.deficit = state.deficit.min(0) + QUANTUM; - if !state.ringed { - state.ringed = true; - self.levels.entry(state.priority).or_default().push_back(group); + if state.ring.is_none() { + self.ring(group); } self.backlogged += 1; } @@ -595,8 +598,8 @@ impl Schedule { /// Account for a group whose queue may have run dry. /// /// A group heading its round robin, as one just served does, leaves it now. - /// Anywhere else its entry stays until the round robin reaches it, since finding it - /// would mean a scan. + /// Anywhere else it keeps its place until the round robin reaches it, so a group that + /// refills before then is not sent to the back. fn deactivate(&mut self, group: Option) { let Some(group) = group else { return; @@ -606,18 +609,11 @@ impl Schedule { return; } state.active = false; + self.backlogged -= 1; - if let Some(level) = self.levels.get_mut(&state.priority) - && level.front() == Some(&group) - { - level.pop_front(); - if level.is_empty() { - self.levels.remove(&state.priority); - } - state.ringed = false; + if self.levels.get(&state.priority) == Some(&group) { + self.unring(group); } - - self.backlogged -= 1; } /// Drop a stream's membership of its group. @@ -634,20 +630,55 @@ impl Schedule { return; } - // Nothing names or fills the group, but a stale round-robin entry may remain. - let state = self.groups.remove(group); - if state.ringed { - self.unring_group(group, state.priority); + // Nothing names or fills the group, but it may still hold its round-robin place. + if state.ring.is_some() { + self.unring(group); } + self.groups.remove(group); } - fn unring_group(&mut self, group: usize, priority: i64) { - let level = self.levels.get_mut(&priority).expect("ringed group has a level"); - let index = level.iter().position(|&g| g == group); - level.remove(index.expect("ringed group is in its level")); - if level.is_empty() { + /// Add a group to the back of its level's round robin. + fn ring(&mut self, group: usize) { + debug_assert!(self.groups[group].ring.is_none(), "group ringed twice"); + let ring = match self.levels.entry(self.groups[group].priority) { + btree_map::Entry::Vacant(level) => { + level.insert(group); + Ring { + prev: group, + next: group, + } + } + btree_map::Entry::Occupied(level) => { + let head = *level.get(); + let tail = self.link(head).prev; + self.link(tail).next = group; + self.link(head).prev = group; + Ring { prev: tail, next: head } + } + }; + self.groups[group].ring = Some(ring); + } + + /// Take a group out of its level's round robin, wherever it sits. + fn unring(&mut self, group: usize) { + let state = &mut self.groups[group]; + let Ring { prev, next } = state.ring.take().expect("group is ringed"); + let priority = state.priority; + if next == group { self.levels.remove(&priority); + return; } + self.link(prev).next = next; + self.link(next).prev = prev; + let head = self.levels.get_mut(&priority).expect("ringed group has a level"); + if *head == group { + *head = next; + } + } + + /// The round-robin neighbors of a ringed group. + fn link(&mut self, group: usize) -> &mut Ring { + self.groups[group].ring.as_mut().expect("group is ringed") } } @@ -814,6 +845,25 @@ mod tests { assert!(ids.contains(&first) && ids.contains(&second), "{ids:?}"); } + #[test] + fn groups_leave_the_round_robin_from_anywhere() { + let mut sim = Sim::new(); + let groups: Vec<_> = (0..4).map(|_| sim.schedule.open_group(0)).collect(); + let ids: Vec<_> = groups.iter().map(|&g| sim.stream(Some(g), 0, 100_000)).collect(); + + // Retire the third group from the middle of the round robin. + let mut stream = sim.streams.remove(&ids[2]).unwrap(); + sim.schedule.close_group(groups[2]); + sim.schedule.leave(&mut stream.slot); + // The second leaves for a level of its own, then returns behind the rest. + sim.schedule.set_group_priority(groups[1], 1); + sim.schedule.set_group_priority(groups[1], 0); + + let order: Vec<_> = (0..6).map(|_| sim.step(QUANTUM as usize).unwrap()).collect(); + assert_eq!(order, [ids[0], ids[3], ids[1], ids[0], ids[3], ids[1]]); + assert!(sim.schedule.is_clean()); + } + #[test] fn datagrams_follow_their_order_and_win_a_tie() { let mut sim = Sim::new();