From 321e18dd1b7251f672f05d71f396848ccd60df0d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:28:38 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m2/perf/kio-channel-contention/spike-cluster.md Quest-Claim: 98ce36a6deb1b29d From ddd8b042ab82c7a4d53957e31fe35942957b7dfb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 02:00:48 -0800 Subject: [PATCH 2/5] spike(relay): a cluster per worker over the in-memory transport Throwaway: SPIKE_CLUSTER=1 gives each tokio worker its own Cluster and origin, meshed over moq-lite-06 memory sessions. The spike_announce example measures announce delay, frame delay across the hop, and announce table memory. bench/relay.sh gains BENCH_PIN and SPIKE_PERF_STAT, and bench/profile.py SPIKE_PERF_BUFFER. Co-Authored-By: Claude Opus 5.5 --- bench/profile.py | 2 +- bench/relay.sh | 13 +- rs/moq-relay/examples/spike_announce.rs | 386 ++++++++++++++++++++++++ rs/moq-relay/src/lib.rs | 1 + rs/moq-relay/src/relay.rs | 20 +- rs/moq-relay/src/spike.rs | 91 ++++++ 6 files changed, 509 insertions(+), 4 deletions(-) create mode 100644 rs/moq-relay/examples/spike_announce.rs create mode 100644 rs/moq-relay/src/spike.rs diff --git a/bench/profile.py b/bench/profile.py index c483a6d337f..1dfdc216e08 100644 --- a/bench/profile.py +++ b/bench/profile.py @@ -177,7 +177,7 @@ def lock(pid, delay, duration, directory): # DWARF unwinding keeps it. Switch timestamps bound CPU and open waits alike. # The default 512 KiB buffer drops samples at 16 workers; 2 MiB does not. subprocess.run(['perf', 'record', '--quiet', '--no-buildid-cache', '-e', EVENT, '-c', '1', '--call-graph', 'dwarf', '--switch-events', - '-k', 'monotonic', '-m', '2M', '-p', str(pid), '-o', str(data), '--', 'sleep', str(duration)], check=True) + '-k', 'monotonic', '-m', os.environ.get('SPIKE_PERF_BUFFER', '2M'), '-p', str(pid), '-o', str(data), '--', 'sleep', str(duration)], check=True) after = schedstat(pid, names) result, folded = reduce(names, parse(perf_script(data, '--show-switch-events')), before, after) (directory / 'lock.json').write_text(json.dumps(result, indent=2) + '\n') diff --git a/bench/relay.sh b/bench/relay.sh index 562c406d846..7ba73877f71 100755 --- a/bench/relay.sh +++ b/bench/relay.sh @@ -38,7 +38,7 @@ relay_config() { '' '[runtime]' "workers = $workers" - 'pin = false' + "pin = ${BENCH_PIN:-false}" ) ;; io-uring-workers) @@ -305,6 +305,13 @@ run_workload() { HOST_PID=$! # The second half of the host window, matching the summary's throughput window. # Its own process group, so cleanup also stops perf and its workload. + # Spike: count the relay's instructions over the same steady window as the profile. + local stat_pid= + if [[ -n ${SPIKE_PERF_STAT:-} && -z $PROFILE ]]; then + (sleep 5 && exec perf stat -x, -e instructions:u,instructions:k,cycles:u,cycles:k \ + -p "$relay_pid" -o "$directory/perfstat.csv" -- sleep 5) >"$directory/perfstat.log" 2>&1 & + stat_pid=$! + fi case $PROFILE in lock | cpu) setsid python3 "$ROOT/bench/profile.py" "$PROFILE" "$relay_pid" "$profile" --delay 5 --duration 5 \ @@ -326,6 +333,10 @@ run_workload() { return 1 fi HOST_PID= + if [[ -n $stat_pid ]] && ! wait "$stat_pid"; then + cat "$directory/perfstat.log" >&2 + return 1 + fi if [[ -n $PROFILE ]]; then if ! wait "$PROFILE_PID"; then cat "$directory/profile.log" >&2 diff --git a/rs/moq-relay/examples/spike_announce.rs b/rs/moq-relay/examples/spike_announce.rs new file mode 100644 index 00000000000..3df1fc98baf --- /dev/null +++ b/rs/moq-relay/examples/spike_announce.rs @@ -0,0 +1,386 @@ +//! Throwaway spike (never merges): announce propagation delay and announce table +//! memory for W per-thread origins meshed over the in-memory transport, the +//! control plane of the cluster spike (`SPIKE_CLUSTER=1`). +//! +//! Each worker is a thread with a current-thread runtime and its own origin, +//! linked to every other worker by a moq-lite-06 memory session that publishes +//! and subscribes like a dialed cluster peer (`src/spike.rs`). +//! +//! - Delay: one broadcast at a time, published on worker `k % W`; the delay is +//! each other worker's `Start` minus the publisher's own local `Start`. +//! - Memory: N broadcasts spread over the workers, then the process's live heap +//! (a counting allocator) once every worker has seen all N and the heap settles. +//! +//! `cargo run --release -p moq-relay --example spike_announce -- [workers...]` + +use std::{ + alloc::{GlobalAlloc, Layout, System}, + sync::atomic::{AtomicIsize, AtomicU64, Ordering}, + time::{Duration, Instant}, +}; + +use moq_net::{origin, transport::memory}; + +struct Counting; + +static LIVE: AtomicIsize = AtomicIsize::new(0); +static ALLOCS: AtomicU64 = AtomicU64::new(0); + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + LIVE.fetch_add(layout.size() as isize, Ordering::Relaxed); + ALLOCS.fetch_add(1, Ordering::Relaxed); + unsafe { System.alloc(layout) } + } + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + LIVE.fetch_sub(layout.size() as isize, Ordering::Relaxed); + unsafe { System.dealloc(ptr, layout) } + } + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new: usize) -> *mut u8 { + LIVE.fetch_add(new as isize - layout.size() as isize, Ordering::Relaxed); + ALLOCS.fetch_add(1, Ordering::Relaxed); + unsafe { System.realloc(ptr, layout, new) } + } +} + +#[global_allocator] +static GLOBAL: Counting = Counting; + +enum Command { + Publish(String), + /// Publish the `live` broadcast with one track here. + Live, + /// Subscribe to `live/t` here, reporting each frame's arrival. + Watch, + /// Write a one-frame group carrying this sequence number, reporting its write time. + Write(u64), + Stop, +} + +struct Delivered { + worker: usize, + sequence: u64, + at: Instant, +} + +struct Seen { + worker: usize, + path: String, + at: Instant, +} + +struct End { + dial: bool, + session: memory::Session, +} + +fn mesh(workers: usize) -> Vec> { + let mut ends: Vec> = (0..workers).map(|_| Vec::new()).collect(); + for low in 0..workers { + for high in low + 1..workers { + let mut config = memory::Config::default(); + config.protocol = Some("moq-lite-06".into()); + let (client, server) = memory::pair(config); + ends[low].push(End { + dial: true, + session: client, + }); + ends[high].push(End { + dial: false, + session: server, + }); + } + } + ends +} + +async fn link(origin: origin::Producer, end: End) { + let origin = origin.with_link(origin::Link::Peer); + let publish = origin.consume().with_hidden(true); + let now = moq_net::time::Instant::now(); + let started = match end.dial { + true => { + moq_net::Client::new() + .with_publisher(publish) + .with_subscriber(origin) + .connect(now, end.session) + .await + } + false => { + moq_net::Server::new() + .with_publisher(publish) + .with_subscriber(origin) + .accept(now, end.session) + .await + } + }; + let (session, driver) = started.expect("mesh link handshake"); + moq_net::time::run(driver).await; + drop(session); +} + +fn worker( + index: usize, + ends: Vec, + mut commands: tokio::sync::mpsc::UnboundedReceiver, + seen: std::sync::mpsc::Sender, + delivered: std::sync::mpsc::Sender, +) { + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + let local = tokio::task::LocalSet::new(); + local.block_on(&runtime, async move { + let (origin, driver) = origin::Producer::new(origin::Config::default()); + tokio::task::spawn_local(moq_net::time::run(driver)); + for end in ends { + tokio::task::spawn_local(link(origin.clone(), end)); + } + let mut announced = origin.consume().announced(); + tokio::task::spawn_local(async move { + while let Some(event) = announced.next().await { + if let moq_net::announce::Event::Start(announce) = event { + let _ = seen.send(Seen { + worker: index, + path: announce.prefix.to_string(), + at: Instant::now(), + }); + } + } + }); + let mut held = Vec::new(); + let mut live = None; + while let Some(command) = commands.recv().await { + match command { + Command::Publish(path) => held.push(origin.publish(path.as_str(), Default::default()).unwrap()), + Command::Live => { + let broadcast = origin.publish("live", Default::default()).unwrap(); + let track = broadcast.create_track("t", None).unwrap(); + live = Some((broadcast, track)); + } + Command::Watch => { + let consumer = origin.consume(); + let delivered = delivered.clone(); + tokio::task::spawn_local(async move { + let broadcast = consumer.request("live", None).await.expect("live resolves"); + let mut subscriber = broadcast + .track("t") + .unwrap() + .subscribe(None) + .await + .expect("subscribe live"); + while let Ok(Some(mut group)) = subscriber.recv_group().await { + while let Ok(Some(frame)) = group.read_frame().await { + let at = Instant::now(); + let sequence = u64::from_le_bytes(frame.payload[..8].try_into().unwrap()); + let _ = delivered.send(Delivered { + worker: index, + sequence, + at, + }); + } + } + }); + } + Command::Write(sequence) => { + let (_, track) = live.as_mut().expect("live published here"); + let at = Instant::now(); + let mut payload = sequence.to_le_bytes().to_vec(); + payload.resize(300, 0); + track.write_frame(None, payload).unwrap(); + let _ = delivered.send(Delivered { + worker: usize::MAX, + sequence, + at, + }); + } + Command::Stop => break, + } + } + drop(live); + drop(held); + }); +} + +/// Wait until every worker has seen each of `paths`; returns each path's per-worker arrival. +fn await_all( + seen: &std::sync::mpsc::Receiver, + workers: usize, + paths: &[String], +) -> std::collections::HashMap>> { + let mut arrivals: std::collections::HashMap>> = + paths.iter().map(|path| (path.clone(), vec![None; workers])).collect(); + let mut missing = paths.len() * workers; + while missing > 0 { + let event = seen.recv_timeout(Duration::from_secs(30)).expect("announce never arrived"); + if let Some(slots) = arrivals.get_mut(&event.path) + && slots[event.worker].is_none() + { + slots[event.worker] = Some(event.at); + missing -= 1; + } + } + arrivals +} + +/// Live heap once it stops moving for 500 ms. +fn settled() -> isize { + let mut last = LIVE.load(Ordering::Relaxed); + loop { + std::thread::sleep(Duration::from_millis(500)); + let now = LIVE.load(Ordering::Relaxed); + if (now - last).abs() <= last / 1000 { + return now; + } + last = now; + } +} + +fn percentile(sorted: &[Duration], p: f64) -> Duration { + sorted[((sorted.len() - 1) as f64 * p).round() as usize] +} + +fn run(workers: usize, rounds: usize, broadcasts: usize) { + let (seen_tx, seen) = std::sync::mpsc::channel(); + let (delivered_tx, delivered) = std::sync::mpsc::channel(); + let mut senders = Vec::new(); + let mut threads = Vec::new(); + for (index, ends) in mesh(workers).into_iter().enumerate() { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + senders.push(tx); + let seen = seen_tx.clone(); + let delivered = delivered_tx.clone(); + threads.push(std::thread::spawn(move || worker(index, ends, rx, seen, delivered))); + } + + // Warm up: every link up and passing announcements. + senders[0].send(Command::Publish("warmup".into())).unwrap(); + await_all(&seen, workers, &["warmup".into()]); + + // Delay, one broadcast at a time. + let mut delays = Vec::new(); + for round in 0..rounds { + let path = format!("delay/{round}"); + let publisher = round % workers; + senders[publisher].send(Command::Publish(path.clone())).unwrap(); + let arrivals = await_all(&seen, workers, std::slice::from_ref(&path)); + let slots = &arrivals[&path]; + let start = slots[publisher].unwrap(); + for (worker, at) in slots.iter().enumerate() { + if worker != publisher { + delays.push(at.unwrap() - start); + } + } + } + + // Frame delivery: one-frame groups written on worker 0, read by a subscriber on + // every worker; worker 0's own subscriber is the shared-origin reference. + senders[0].send(Command::Live).unwrap(); + await_all(&seen, workers, &["live".into()]); + for sender in &senders { + sender.send(Command::Watch).unwrap(); + } + std::thread::sleep(Duration::from_millis(200)); + let mut local = Vec::new(); + let mut remote = Vec::new(); + for sequence in 0..rounds as u64 { + senders[0].send(Command::Write(sequence)).unwrap(); + let mut written = None; + let mut arrivals = vec![None; workers]; + while written.is_none() || arrivals.iter().any(Option::is_none) { + let event = delivered.recv_timeout(Duration::from_secs(30)).expect("frame never arrived"); + if event.sequence != sequence { + continue; + } + match event.worker { + usize::MAX => written = Some(event.at), + worker => arrivals[worker] = Some(event.at), + } + } + let written = written.unwrap(); + for (worker, at) in arrivals.into_iter().enumerate() { + let delay = at.unwrap().saturating_duration_since(written); + match worker { + 0 => local.push(delay), + _ => remote.push(delay), + } + } + std::thread::sleep(Duration::from_millis(2)); + } + local.sort(); + remote.sort(); + let us = |d: Duration| d.as_secs_f64() * 1e6; + match remote.is_empty() { + true => eprintln!( + "W={workers} frame delay local p50 {:.1} p99 {:.1} us", + us(percentile(&local, 0.5)), + us(percentile(&local, 0.99)) + ), + false => eprintln!( + "W={workers} frame delay local p50 {:.1} p99 {:.1} us; across the hop p50 {:.1} p99 {:.1} max {:.1} us", + us(percentile(&local, 0.5)), + us(percentile(&local, 0.99)), + us(percentile(&remote, 0.5)), + us(percentile(&remote, 0.99)), + us(*remote.last().unwrap()), + ), + } + + // Memory and convergence, N broadcasts at once. + let before = settled(); + let allocs = ALLOCS.load(Ordering::Relaxed); + let paths: Vec = (0..broadcasts).map(|i| format!("mem/{i}")).collect(); + let started = Instant::now(); + for (i, path) in paths.iter().enumerate() { + senders[i % workers].send(Command::Publish(path.clone())).unwrap(); + } + let arrivals = await_all(&seen, workers, &paths); + let converged = arrivals + .values() + .flat_map(|slots| slots.iter().map(|at| at.unwrap())) + .max() + .unwrap() + - started; + let after = settled(); + let allocs = ALLOCS.load(Ordering::Relaxed) - allocs; + + delays.sort(); + let per = (after - before) as f64 / broadcasts as f64; + match delays.is_empty() { + true => print!("{workers:>3} {:>9} {:>9} {:>9}", "-", "-", "-"), + false => print!( + "{workers:>3} {:>9.1} {:>9.1} {:>9.1}", + percentile(&delays, 0.5).as_secs_f64() * 1e6, + percentile(&delays, 0.99).as_secs_f64() * 1e6, + delays.last().unwrap().as_secs_f64() * 1e6, + ), + } + println!( + " {broadcasts:>7} {:>10.1} {:>12.0} {:>12.0} {:>12.1}", + converged.as_secs_f64() * 1e3, + per, + per / workers as f64, + allocs as f64 / broadcasts as f64, + ); + + for sender in &senders { + let _ = sender.send(Command::Stop); + } + for thread in threads { + let _ = thread.join(); + } +} + +fn main() { + let mut args = std::env::args().skip(1).map(|arg| arg.parse::().unwrap()); + let broadcasts = args.next().unwrap_or(10_000); + let workers: Vec = args.collect(); + let workers = match workers.is_empty() { + true => vec![1, 2, 4, 8, 16], + false => workers, + }; + println!( + " W p50(us) p99(us) max(us) N conv(ms) heap B/bcast per worker allocs/bcast" + ); + for w in workers { + run(w, 200, broadcasts); + } +} diff --git a/rs/moq-relay/src/lib.rs b/rs/moq-relay/src/lib.rs index fd896f264e6..36db6ae1627 100644 --- a/rs/moq-relay/src/lib.rs +++ b/rs/moq-relay/src/lib.rs @@ -32,6 +32,7 @@ mod relay; pub mod runtime; pub mod session; mod settings; +mod spike; pub mod shutdown; pub mod stats; #[cfg(any(feature = "stun", test))] diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index 44a90d1e2d2..12431dd6e60 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -574,6 +574,7 @@ impl Relay { workers, #[cfg(all(target_os = "linux", feature = "_uring"))] uring, + config, .. } = self; @@ -684,13 +685,28 @@ impl Relay { let quic_workers = { let mut running = futures::stream::FuturesUnordered::new(); if let Some(workers) = workers.as_mut() { - for member in workers.members() { + let members = workers.members(); + let mut mesh = match crate::spike::enabled() { + true => { + tracing::warn!(workers = members.len(), "spike: a cluster per worker, meshed in memory"); + crate::spike::mesh(members.len()).into_iter().map(Some).collect() + } + false => Vec::new(), + }; + for member in members { let index = member.index(); let cluster = cluster.clone(); let auth = auth.clone(); let worker_shutdown = shutdown.clone(); let sessions = sessions.clone(); - let task = member.serve(move |server| serve(server, cluster, auth, worker_shutdown, sessions)); + let spike = mesh.get_mut(usize::from(index)).and_then(Option::take).map(|ends| (config.clone(), ends)); + let task = member.serve(move |server| async move { + let cluster = match spike { + Some((config, ends)) => crate::spike::worker(&config, ends)?, + None => cluster, + }; + serve(server, cluster, auth, worker_shutdown, sessions).await + }); running.push(async move { match task.await { Ok(res) => res.with_context(|| format!("QUIC worker {index} failed")), diff --git a/rs/moq-relay/src/spike.rs b/rs/moq-relay/src/spike.rs new file mode 100644 index 00000000000..80281763a53 --- /dev/null +++ b/rs/moq-relay/src/spike.rs @@ -0,0 +1,91 @@ +//! Throwaway spike (never merges): a relay per worker thread, clustered over the +//! in-memory transport with nothing shared. +//! +//! `SPIKE_CLUSTER=1` with `runtime.workers` gives each tokio worker its own +//! [`Cluster`](crate::cluster::Cluster): its own origin, cache pool, and origin +//! driver, built on the worker's thread. The workers link in a full mesh of +//! [`moq_net::transport::memory`] sessions, so announcements, routes, and groups +//! all cross workers through moq-lite, exactly as between clustered relays. +//! The lower-indexed worker of each pair dials and the other accepts, and each +//! link publishes and subscribes like a dialed cluster peer +//! (`Cluster::run_remote_session`). + +use moq_net::{origin, transport::memory}; + +/// Whether the spike is on. +pub(crate) fn enabled() -> bool { + std::env::var_os("SPIKE_CLUSTER").is_some_and(|value| !value.is_empty() && value != "0") +} + +/// One worker's end of a mesh link. +pub(crate) struct End { + dial: bool, + session: memory::Session, +} + +/// Every worker's link ends, indexed by worker. +pub(crate) fn mesh(workers: usize) -> Vec> { + let mut ends: Vec> = (0..workers).map(|_| Vec::new()).collect(); + for low in 0..workers { + for high in low + 1..workers { + let mut config = memory::Config::default(); + config.protocol = Some("moq-lite-06".into()); + let (client, server) = memory::pair(config); + ends[low].push(End { + dial: true, + session: client, + }); + ends[high].push(End { + dial: false, + session: server, + }); + } + } + ends +} + +/// Build this worker's own cluster and start its mesh links on the current thread. +pub(crate) fn worker(config: &crate::Config, ends: Vec) -> anyhow::Result { + let cache = config.cache.init()?; + let mut options = crate::cluster::Options::new(config.cluster.clone()).with_cache(cache); + options.session = config.limits.client.clone(); + options.peer = config.limits.peer.clone(); + let limits = options.peer.resolve(); + let cluster = crate::cluster::Cluster::new(options)?; + for end in ends { + let origin = cluster.origin.clone(); + tokio::task::spawn_local(async move { + if let Err(err) = link(origin, limits, end).await { + tracing::warn!(%err, "spike mesh link ended"); + } + }); + } + Ok(cluster) +} + +async fn link(origin: origin::Producer, limits: moq_net::session::Limits, end: End) -> anyhow::Result<()> { + let origin = origin.with_link(origin::Link::Peer); + let publish = origin.consume().with_hidden(true); + let now = moq_net::time::Instant::now(); + let (session, driver) = match end.dial { + true => { + moq_net::Client::new() + .with_publisher(publish) + .with_subscriber(origin) + .with_limits(limits) + .connect(now, end.session) + .await? + } + false => { + moq_net::Server::new() + .with_publisher(publish) + .with_subscriber(origin) + .with_limits(limits) + .accept(now, end.session) + .await? + } + }; + let err = moq_net::time::run(driver).await; + drop(session); + Err(err.into()) +} From 3c109d34df1f76b24ffab71e0cac9c28eccc09d1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 02:34:06 -0800 Subject: [PATCH 3/5] spike(relay): remove the cluster spike harness The spike never merges; its prototype lives in ddd8b042ab. Co-Authored-By: Claude Opus 5.5 --- bench/profile.py | 2 +- bench/relay.sh | 13 +- rs/moq-relay/examples/spike_announce.rs | 386 ------------------------ rs/moq-relay/src/lib.rs | 1 - rs/moq-relay/src/relay.rs | 20 +- rs/moq-relay/src/spike.rs | 91 ------ 6 files changed, 4 insertions(+), 509 deletions(-) delete mode 100644 rs/moq-relay/examples/spike_announce.rs delete mode 100644 rs/moq-relay/src/spike.rs diff --git a/bench/profile.py b/bench/profile.py index 1dfdc216e08..c483a6d337f 100644 --- a/bench/profile.py +++ b/bench/profile.py @@ -177,7 +177,7 @@ def lock(pid, delay, duration, directory): # DWARF unwinding keeps it. Switch timestamps bound CPU and open waits alike. # The default 512 KiB buffer drops samples at 16 workers; 2 MiB does not. subprocess.run(['perf', 'record', '--quiet', '--no-buildid-cache', '-e', EVENT, '-c', '1', '--call-graph', 'dwarf', '--switch-events', - '-k', 'monotonic', '-m', os.environ.get('SPIKE_PERF_BUFFER', '2M'), '-p', str(pid), '-o', str(data), '--', 'sleep', str(duration)], check=True) + '-k', 'monotonic', '-m', '2M', '-p', str(pid), '-o', str(data), '--', 'sleep', str(duration)], check=True) after = schedstat(pid, names) result, folded = reduce(names, parse(perf_script(data, '--show-switch-events')), before, after) (directory / 'lock.json').write_text(json.dumps(result, indent=2) + '\n') diff --git a/bench/relay.sh b/bench/relay.sh index 7ba73877f71..562c406d846 100755 --- a/bench/relay.sh +++ b/bench/relay.sh @@ -38,7 +38,7 @@ relay_config() { '' '[runtime]' "workers = $workers" - "pin = ${BENCH_PIN:-false}" + 'pin = false' ) ;; io-uring-workers) @@ -305,13 +305,6 @@ run_workload() { HOST_PID=$! # The second half of the host window, matching the summary's throughput window. # Its own process group, so cleanup also stops perf and its workload. - # Spike: count the relay's instructions over the same steady window as the profile. - local stat_pid= - if [[ -n ${SPIKE_PERF_STAT:-} && -z $PROFILE ]]; then - (sleep 5 && exec perf stat -x, -e instructions:u,instructions:k,cycles:u,cycles:k \ - -p "$relay_pid" -o "$directory/perfstat.csv" -- sleep 5) >"$directory/perfstat.log" 2>&1 & - stat_pid=$! - fi case $PROFILE in lock | cpu) setsid python3 "$ROOT/bench/profile.py" "$PROFILE" "$relay_pid" "$profile" --delay 5 --duration 5 \ @@ -333,10 +326,6 @@ run_workload() { return 1 fi HOST_PID= - if [[ -n $stat_pid ]] && ! wait "$stat_pid"; then - cat "$directory/perfstat.log" >&2 - return 1 - fi if [[ -n $PROFILE ]]; then if ! wait "$PROFILE_PID"; then cat "$directory/profile.log" >&2 diff --git a/rs/moq-relay/examples/spike_announce.rs b/rs/moq-relay/examples/spike_announce.rs deleted file mode 100644 index 3df1fc98baf..00000000000 --- a/rs/moq-relay/examples/spike_announce.rs +++ /dev/null @@ -1,386 +0,0 @@ -//! Throwaway spike (never merges): announce propagation delay and announce table -//! memory for W per-thread origins meshed over the in-memory transport, the -//! control plane of the cluster spike (`SPIKE_CLUSTER=1`). -//! -//! Each worker is a thread with a current-thread runtime and its own origin, -//! linked to every other worker by a moq-lite-06 memory session that publishes -//! and subscribes like a dialed cluster peer (`src/spike.rs`). -//! -//! - Delay: one broadcast at a time, published on worker `k % W`; the delay is -//! each other worker's `Start` minus the publisher's own local `Start`. -//! - Memory: N broadcasts spread over the workers, then the process's live heap -//! (a counting allocator) once every worker has seen all N and the heap settles. -//! -//! `cargo run --release -p moq-relay --example spike_announce -- [workers...]` - -use std::{ - alloc::{GlobalAlloc, Layout, System}, - sync::atomic::{AtomicIsize, AtomicU64, Ordering}, - time::{Duration, Instant}, -}; - -use moq_net::{origin, transport::memory}; - -struct Counting; - -static LIVE: AtomicIsize = AtomicIsize::new(0); -static ALLOCS: AtomicU64 = AtomicU64::new(0); - -unsafe impl GlobalAlloc for Counting { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - LIVE.fetch_add(layout.size() as isize, Ordering::Relaxed); - ALLOCS.fetch_add(1, Ordering::Relaxed); - unsafe { System.alloc(layout) } - } - unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - LIVE.fetch_sub(layout.size() as isize, Ordering::Relaxed); - unsafe { System.dealloc(ptr, layout) } - } - unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new: usize) -> *mut u8 { - LIVE.fetch_add(new as isize - layout.size() as isize, Ordering::Relaxed); - ALLOCS.fetch_add(1, Ordering::Relaxed); - unsafe { System.realloc(ptr, layout, new) } - } -} - -#[global_allocator] -static GLOBAL: Counting = Counting; - -enum Command { - Publish(String), - /// Publish the `live` broadcast with one track here. - Live, - /// Subscribe to `live/t` here, reporting each frame's arrival. - Watch, - /// Write a one-frame group carrying this sequence number, reporting its write time. - Write(u64), - Stop, -} - -struct Delivered { - worker: usize, - sequence: u64, - at: Instant, -} - -struct Seen { - worker: usize, - path: String, - at: Instant, -} - -struct End { - dial: bool, - session: memory::Session, -} - -fn mesh(workers: usize) -> Vec> { - let mut ends: Vec> = (0..workers).map(|_| Vec::new()).collect(); - for low in 0..workers { - for high in low + 1..workers { - let mut config = memory::Config::default(); - config.protocol = Some("moq-lite-06".into()); - let (client, server) = memory::pair(config); - ends[low].push(End { - dial: true, - session: client, - }); - ends[high].push(End { - dial: false, - session: server, - }); - } - } - ends -} - -async fn link(origin: origin::Producer, end: End) { - let origin = origin.with_link(origin::Link::Peer); - let publish = origin.consume().with_hidden(true); - let now = moq_net::time::Instant::now(); - let started = match end.dial { - true => { - moq_net::Client::new() - .with_publisher(publish) - .with_subscriber(origin) - .connect(now, end.session) - .await - } - false => { - moq_net::Server::new() - .with_publisher(publish) - .with_subscriber(origin) - .accept(now, end.session) - .await - } - }; - let (session, driver) = started.expect("mesh link handshake"); - moq_net::time::run(driver).await; - drop(session); -} - -fn worker( - index: usize, - ends: Vec, - mut commands: tokio::sync::mpsc::UnboundedReceiver, - seen: std::sync::mpsc::Sender, - delivered: std::sync::mpsc::Sender, -) { - let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); - let local = tokio::task::LocalSet::new(); - local.block_on(&runtime, async move { - let (origin, driver) = origin::Producer::new(origin::Config::default()); - tokio::task::spawn_local(moq_net::time::run(driver)); - for end in ends { - tokio::task::spawn_local(link(origin.clone(), end)); - } - let mut announced = origin.consume().announced(); - tokio::task::spawn_local(async move { - while let Some(event) = announced.next().await { - if let moq_net::announce::Event::Start(announce) = event { - let _ = seen.send(Seen { - worker: index, - path: announce.prefix.to_string(), - at: Instant::now(), - }); - } - } - }); - let mut held = Vec::new(); - let mut live = None; - while let Some(command) = commands.recv().await { - match command { - Command::Publish(path) => held.push(origin.publish(path.as_str(), Default::default()).unwrap()), - Command::Live => { - let broadcast = origin.publish("live", Default::default()).unwrap(); - let track = broadcast.create_track("t", None).unwrap(); - live = Some((broadcast, track)); - } - Command::Watch => { - let consumer = origin.consume(); - let delivered = delivered.clone(); - tokio::task::spawn_local(async move { - let broadcast = consumer.request("live", None).await.expect("live resolves"); - let mut subscriber = broadcast - .track("t") - .unwrap() - .subscribe(None) - .await - .expect("subscribe live"); - while let Ok(Some(mut group)) = subscriber.recv_group().await { - while let Ok(Some(frame)) = group.read_frame().await { - let at = Instant::now(); - let sequence = u64::from_le_bytes(frame.payload[..8].try_into().unwrap()); - let _ = delivered.send(Delivered { - worker: index, - sequence, - at, - }); - } - } - }); - } - Command::Write(sequence) => { - let (_, track) = live.as_mut().expect("live published here"); - let at = Instant::now(); - let mut payload = sequence.to_le_bytes().to_vec(); - payload.resize(300, 0); - track.write_frame(None, payload).unwrap(); - let _ = delivered.send(Delivered { - worker: usize::MAX, - sequence, - at, - }); - } - Command::Stop => break, - } - } - drop(live); - drop(held); - }); -} - -/// Wait until every worker has seen each of `paths`; returns each path's per-worker arrival. -fn await_all( - seen: &std::sync::mpsc::Receiver, - workers: usize, - paths: &[String], -) -> std::collections::HashMap>> { - let mut arrivals: std::collections::HashMap>> = - paths.iter().map(|path| (path.clone(), vec![None; workers])).collect(); - let mut missing = paths.len() * workers; - while missing > 0 { - let event = seen.recv_timeout(Duration::from_secs(30)).expect("announce never arrived"); - if let Some(slots) = arrivals.get_mut(&event.path) - && slots[event.worker].is_none() - { - slots[event.worker] = Some(event.at); - missing -= 1; - } - } - arrivals -} - -/// Live heap once it stops moving for 500 ms. -fn settled() -> isize { - let mut last = LIVE.load(Ordering::Relaxed); - loop { - std::thread::sleep(Duration::from_millis(500)); - let now = LIVE.load(Ordering::Relaxed); - if (now - last).abs() <= last / 1000 { - return now; - } - last = now; - } -} - -fn percentile(sorted: &[Duration], p: f64) -> Duration { - sorted[((sorted.len() - 1) as f64 * p).round() as usize] -} - -fn run(workers: usize, rounds: usize, broadcasts: usize) { - let (seen_tx, seen) = std::sync::mpsc::channel(); - let (delivered_tx, delivered) = std::sync::mpsc::channel(); - let mut senders = Vec::new(); - let mut threads = Vec::new(); - for (index, ends) in mesh(workers).into_iter().enumerate() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - senders.push(tx); - let seen = seen_tx.clone(); - let delivered = delivered_tx.clone(); - threads.push(std::thread::spawn(move || worker(index, ends, rx, seen, delivered))); - } - - // Warm up: every link up and passing announcements. - senders[0].send(Command::Publish("warmup".into())).unwrap(); - await_all(&seen, workers, &["warmup".into()]); - - // Delay, one broadcast at a time. - let mut delays = Vec::new(); - for round in 0..rounds { - let path = format!("delay/{round}"); - let publisher = round % workers; - senders[publisher].send(Command::Publish(path.clone())).unwrap(); - let arrivals = await_all(&seen, workers, std::slice::from_ref(&path)); - let slots = &arrivals[&path]; - let start = slots[publisher].unwrap(); - for (worker, at) in slots.iter().enumerate() { - if worker != publisher { - delays.push(at.unwrap() - start); - } - } - } - - // Frame delivery: one-frame groups written on worker 0, read by a subscriber on - // every worker; worker 0's own subscriber is the shared-origin reference. - senders[0].send(Command::Live).unwrap(); - await_all(&seen, workers, &["live".into()]); - for sender in &senders { - sender.send(Command::Watch).unwrap(); - } - std::thread::sleep(Duration::from_millis(200)); - let mut local = Vec::new(); - let mut remote = Vec::new(); - for sequence in 0..rounds as u64 { - senders[0].send(Command::Write(sequence)).unwrap(); - let mut written = None; - let mut arrivals = vec![None; workers]; - while written.is_none() || arrivals.iter().any(Option::is_none) { - let event = delivered.recv_timeout(Duration::from_secs(30)).expect("frame never arrived"); - if event.sequence != sequence { - continue; - } - match event.worker { - usize::MAX => written = Some(event.at), - worker => arrivals[worker] = Some(event.at), - } - } - let written = written.unwrap(); - for (worker, at) in arrivals.into_iter().enumerate() { - let delay = at.unwrap().saturating_duration_since(written); - match worker { - 0 => local.push(delay), - _ => remote.push(delay), - } - } - std::thread::sleep(Duration::from_millis(2)); - } - local.sort(); - remote.sort(); - let us = |d: Duration| d.as_secs_f64() * 1e6; - match remote.is_empty() { - true => eprintln!( - "W={workers} frame delay local p50 {:.1} p99 {:.1} us", - us(percentile(&local, 0.5)), - us(percentile(&local, 0.99)) - ), - false => eprintln!( - "W={workers} frame delay local p50 {:.1} p99 {:.1} us; across the hop p50 {:.1} p99 {:.1} max {:.1} us", - us(percentile(&local, 0.5)), - us(percentile(&local, 0.99)), - us(percentile(&remote, 0.5)), - us(percentile(&remote, 0.99)), - us(*remote.last().unwrap()), - ), - } - - // Memory and convergence, N broadcasts at once. - let before = settled(); - let allocs = ALLOCS.load(Ordering::Relaxed); - let paths: Vec = (0..broadcasts).map(|i| format!("mem/{i}")).collect(); - let started = Instant::now(); - for (i, path) in paths.iter().enumerate() { - senders[i % workers].send(Command::Publish(path.clone())).unwrap(); - } - let arrivals = await_all(&seen, workers, &paths); - let converged = arrivals - .values() - .flat_map(|slots| slots.iter().map(|at| at.unwrap())) - .max() - .unwrap() - - started; - let after = settled(); - let allocs = ALLOCS.load(Ordering::Relaxed) - allocs; - - delays.sort(); - let per = (after - before) as f64 / broadcasts as f64; - match delays.is_empty() { - true => print!("{workers:>3} {:>9} {:>9} {:>9}", "-", "-", "-"), - false => print!( - "{workers:>3} {:>9.1} {:>9.1} {:>9.1}", - percentile(&delays, 0.5).as_secs_f64() * 1e6, - percentile(&delays, 0.99).as_secs_f64() * 1e6, - delays.last().unwrap().as_secs_f64() * 1e6, - ), - } - println!( - " {broadcasts:>7} {:>10.1} {:>12.0} {:>12.0} {:>12.1}", - converged.as_secs_f64() * 1e3, - per, - per / workers as f64, - allocs as f64 / broadcasts as f64, - ); - - for sender in &senders { - let _ = sender.send(Command::Stop); - } - for thread in threads { - let _ = thread.join(); - } -} - -fn main() { - let mut args = std::env::args().skip(1).map(|arg| arg.parse::().unwrap()); - let broadcasts = args.next().unwrap_or(10_000); - let workers: Vec = args.collect(); - let workers = match workers.is_empty() { - true => vec![1, 2, 4, 8, 16], - false => workers, - }; - println!( - " W p50(us) p99(us) max(us) N conv(ms) heap B/bcast per worker allocs/bcast" - ); - for w in workers { - run(w, 200, broadcasts); - } -} diff --git a/rs/moq-relay/src/lib.rs b/rs/moq-relay/src/lib.rs index 36db6ae1627..fd896f264e6 100644 --- a/rs/moq-relay/src/lib.rs +++ b/rs/moq-relay/src/lib.rs @@ -32,7 +32,6 @@ mod relay; pub mod runtime; pub mod session; mod settings; -mod spike; pub mod shutdown; pub mod stats; #[cfg(any(feature = "stun", test))] diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index 12431dd6e60..44a90d1e2d2 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -574,7 +574,6 @@ impl Relay { workers, #[cfg(all(target_os = "linux", feature = "_uring"))] uring, - config, .. } = self; @@ -685,28 +684,13 @@ impl Relay { let quic_workers = { let mut running = futures::stream::FuturesUnordered::new(); if let Some(workers) = workers.as_mut() { - let members = workers.members(); - let mut mesh = match crate::spike::enabled() { - true => { - tracing::warn!(workers = members.len(), "spike: a cluster per worker, meshed in memory"); - crate::spike::mesh(members.len()).into_iter().map(Some).collect() - } - false => Vec::new(), - }; - for member in members { + for member in workers.members() { let index = member.index(); let cluster = cluster.clone(); let auth = auth.clone(); let worker_shutdown = shutdown.clone(); let sessions = sessions.clone(); - let spike = mesh.get_mut(usize::from(index)).and_then(Option::take).map(|ends| (config.clone(), ends)); - let task = member.serve(move |server| async move { - let cluster = match spike { - Some((config, ends)) => crate::spike::worker(&config, ends)?, - None => cluster, - }; - serve(server, cluster, auth, worker_shutdown, sessions).await - }); + let task = member.serve(move |server| serve(server, cluster, auth, worker_shutdown, sessions)); running.push(async move { match task.await { Ok(res) => res.with_context(|| format!("QUIC worker {index} failed")), diff --git a/rs/moq-relay/src/spike.rs b/rs/moq-relay/src/spike.rs deleted file mode 100644 index 80281763a53..00000000000 --- a/rs/moq-relay/src/spike.rs +++ /dev/null @@ -1,91 +0,0 @@ -//! Throwaway spike (never merges): a relay per worker thread, clustered over the -//! in-memory transport with nothing shared. -//! -//! `SPIKE_CLUSTER=1` with `runtime.workers` gives each tokio worker its own -//! [`Cluster`](crate::cluster::Cluster): its own origin, cache pool, and origin -//! driver, built on the worker's thread. The workers link in a full mesh of -//! [`moq_net::transport::memory`] sessions, so announcements, routes, and groups -//! all cross workers through moq-lite, exactly as between clustered relays. -//! The lower-indexed worker of each pair dials and the other accepts, and each -//! link publishes and subscribes like a dialed cluster peer -//! (`Cluster::run_remote_session`). - -use moq_net::{origin, transport::memory}; - -/// Whether the spike is on. -pub(crate) fn enabled() -> bool { - std::env::var_os("SPIKE_CLUSTER").is_some_and(|value| !value.is_empty() && value != "0") -} - -/// One worker's end of a mesh link. -pub(crate) struct End { - dial: bool, - session: memory::Session, -} - -/// Every worker's link ends, indexed by worker. -pub(crate) fn mesh(workers: usize) -> Vec> { - let mut ends: Vec> = (0..workers).map(|_| Vec::new()).collect(); - for low in 0..workers { - for high in low + 1..workers { - let mut config = memory::Config::default(); - config.protocol = Some("moq-lite-06".into()); - let (client, server) = memory::pair(config); - ends[low].push(End { - dial: true, - session: client, - }); - ends[high].push(End { - dial: false, - session: server, - }); - } - } - ends -} - -/// Build this worker's own cluster and start its mesh links on the current thread. -pub(crate) fn worker(config: &crate::Config, ends: Vec) -> anyhow::Result { - let cache = config.cache.init()?; - let mut options = crate::cluster::Options::new(config.cluster.clone()).with_cache(cache); - options.session = config.limits.client.clone(); - options.peer = config.limits.peer.clone(); - let limits = options.peer.resolve(); - let cluster = crate::cluster::Cluster::new(options)?; - for end in ends { - let origin = cluster.origin.clone(); - tokio::task::spawn_local(async move { - if let Err(err) = link(origin, limits, end).await { - tracing::warn!(%err, "spike mesh link ended"); - } - }); - } - Ok(cluster) -} - -async fn link(origin: origin::Producer, limits: moq_net::session::Limits, end: End) -> anyhow::Result<()> { - let origin = origin.with_link(origin::Link::Peer); - let publish = origin.consume().with_hidden(true); - let now = moq_net::time::Instant::now(); - let (session, driver) = match end.dial { - true => { - moq_net::Client::new() - .with_publisher(publish) - .with_subscriber(origin) - .with_limits(limits) - .connect(now, end.session) - .await? - } - false => { - moq_net::Server::new() - .with_publisher(publish) - .with_subscriber(origin) - .with_limits(limits) - .accept(now, end.session) - .await? - } - }; - let err = moq_net::time::run(driver).await; - drop(session); - Err(err.into()) -} From 9a2cd375bd3ccf2621a524727b328e64650b2b82 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 02:35:24 -0800 Subject: [PATCH 4/5] quest: report the cluster spike Co-Authored-By: Claude Opus 5.5 --- .../m2/perf/kio-channel-contention/README.md | 83 ++++++++++++++++++- .../kio-channel-contention/log-adoption.md | 1 - .../kio-channel-contention/spike-cluster.md | 28 ------- 3 files changed, 79 insertions(+), 33 deletions(-) delete mode 100644 quest/m2/perf/kio-channel-contention/spike-cluster.md diff --git a/quest/m2/perf/kio-channel-contention/README.md b/quest/m2/perf/kio-channel-contention/README.md index ca1bdcac05e..6bf6d23f3d0 100644 --- a/quest/m2/perf/kio-channel-contention/README.md +++ b/quest/m2/perf/kio-channel-contention/README.md @@ -142,9 +142,8 @@ Decided (maintainer, 2026-10-14, per-thread follow-up): in-memory transport (`moq_net::transport::memory`), never real QUIC. The [replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md) - replicates the data plane and shares announcements; the - [cluster spike](/quest/m2/perf/kio-channel-contention/spike-cluster.md) - shares nothing. A measured win reopens sharding through `/quest-plan`; a + replicates the data plane and shares announcements; the cluster spike + (results below) shares nothing. A measured win reopens sharding through `/quest-plan`; a no-win restores the rule. - Spikes are draft PRs that never merge; their numbers and recommendation land here. The in-memory transport merges for real. @@ -228,9 +227,85 @@ DWARF stacks, 36% incomplete) put waiter registration at 1.97% self-time, the fourth-largest site, on a 501-connection chat shape. It agrees with this epic's targets but carries no lock-wait split. +Cluster spike (2026-11-01, +[moq-dev/moq#7534](https://github.com/moq-dev/moq/pull/7534), prototype at +[`ddd8b042`](https://github.com/moq-dev/moq/commit/ddd8b042ab82c7a4d53957e31fe35942957b7dfb), +not merged): each tokio worker builds its own `Cluster` (origin, cache pool, +origin driver) on its thread, and the workers link in a full mesh of +moq-lite-06 memory sessions that publish and subscribe like a dialed +cluster peer, so announcements, routes, and groups all cross workers +through moq-lite. Measured with `just bench-profile lock`, both arms on the +same build in the same session, plus `perf stat` instruction counts +(user and kernel, both core types) over the same 5 s steady window. The host +stayed loaded (load average 15 to 25), so instructions per delivered frame +rank the arms and wall-clock numbers only corroborate. Workers pinned +(`runtime.pin`) unless noted; two pinned passes (three for pinned chat +`base`), one unpinned. + +Thousand instructions per delivered frame, `base` (today's shared origin) to +`cluster`: + +| shape | 1 worker | 4 workers | 16 workers | 16 workers, unpinned | +| --- | --- | --- | --- | --- | +| chat | 189 to 175 | 219 to 185 (-15%) | 236 to 190 (-20%) | 309 to 191 (-38%) | +| fanout | 188 to 188 | 204 to 200 (-2%) | 255 to 244 (-4%) | 221 to 244 (+10%) | + +Unpinned at 4 workers: chat 229 to 185, fanout 206 to 202. Cells are +medians of the passes; one worker runs the same code in both arms (one +cluster, no links), and chat saturates it, so that column is noise. + +- Lock wait goes to zero: every pinned and unpinned `cluster` run measured + 0 to 2 ms against `base`'s 22 to 710 ms (chat, 4 workers), 2,096 to + 3,159 ms (chat, 16), 106 to 172 ms (fanout, 4), and 669 to 774 ms + (fanout, 16). `base` chat's unpinned lock capture failed three times + (perf saw nonalternating switch records), so that cell has no wait number. +- Relay CPU, chat, pinned: 1.5 to 1.7 cores at 4 workers fell to 1.1, and + 1.8 to 2.4 at 16 fell to 1.2 to 1.3 (unpinned 16: 2.6 to 1.3). Unpinned 4 + workers measured 1.7 in both arms, so CPU time only corroborates the + instruction counts. Fanout stays under 0.08 cores either way. +- The saving is kernel instructions (futex) and off-CPU wait. User + instructions stay flat for chat, but rise 19% for fanout at 16 workers + (173 to 205 thousand per frame): a one-frame group costs about 140 + thousand user instructions per hop, and fanout's 64 subscribers give each + of the 15 hops only four readers to amortize it over. Chat's 2,000 + subscribers amortize it to nothing. +- Resident memory at 16 workers rose 14 MiB for fanout (43 to 57 MiB) and + 20 MiB for chat (507 to 527 MiB): 16 origins and cache pools, 120 links. +- Glass-to-glass p50 stayed within the generator's millisecond resolution. + In process (`spike_announce`, below), a one-frame group read on another + worker arrives 75 to 175 us (p50) and 0.26 to 1.6 ms (p99) after its + write, against 14 to 44 us for a reader on the writing worker. + +The control plane is where it loses. `spike_announce` meshes W origin +threads the same way (on cores 8 to 23, away from the relay runs) and +announces 1,000 broadcasts spread over the workers; heap is a counting +allocator's live bytes once every worker has every broadcast: + +| workers | heap per broadcast | allocations per broadcast | announce delay p50 / p99 | 1,000 broadcasts converge | +| --- | --- | --- | --- | --- | +| 1 | 4.9 KB | 64 | - | 5 to 18 ms | +| 2 | 10.9 KB | 140 | 12 to 18 / 32 to 250 us | 8 to 14 ms | +| 4 | 45 KB | 690 to 730 | 20 to 22 / 63 to 230 us | 31 to 40 ms | +| 8 | 202 to 203 KB | 3,260 to 3,270 | 70 to 80 / 240 to 380 us | 107 to 166 ms | +| 16 | 881 to 889 KB | 14,600 to 16,500 | 290 to 380 / 1,170 to 1,330 us | 0.5 to 0.9 s | + +The heap repeats across runs and grows with W squared: per worker it is +5.4, 11.3, 25.3, and 55.4 KB per broadcast at 2, 4, 8, and 16 workers, about +3.7 KB more per peer. That fits each worker keeping a route per peer, since +a full mesh re-announces every route to every peer (not checked route by +route). At 16 workers a relay with 10,000 broadcasts would hold about 9 GB of +announce state that a shared origin holds in 49 MB. + +Verdict: the data-plane half is a win wherever a track fans out to many +subscribers per worker (chat), and a wash or a loss where it does not +(fanout). Sharing nothing then costs W-squared announce state and +propagation, which grows past any data-plane saving at relay broadcast +counts. Relay-per-thread is not a win as a design; it bounds what the +[replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md), +which keeps one shared announce table, can save on the data plane. + ## Required - [Version counter](/quest/m2/perf/kio-channel-contention/version.md) - an opted-in re-poll skips the mutex when nothing changed, and parking no longer contends with readers - [Replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md) - measures per-worker data-plane replicas linked over the in-memory transport, announcements shared -- [Cluster spike](/quest/m2/perf/kio-channel-contention/spike-cluster.md) - measures a relay per thread clustered over the in-memory transport, nothing shared - [Log adoption](/quest/m2/perf/kio-channel-contention/log-adoption.md) - the rest of moq-net's log-shaped state moves onto the log where it measures as a win diff --git a/quest/m2/perf/kio-channel-contention/log-adoption.md b/quest/m2/perf/kio-channel-contention/log-adoption.md index afe533d180c..37de47ab399 100644 --- a/quest/m2/perf/kio-channel-contention/log-adoption.md +++ b/quest/m2/perf/kio-channel-contention/log-adoption.md @@ -30,4 +30,3 @@ stays on its mutex. ## Required - [Replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md) - if per-worker replicas win, each channel's cross-core readers collapse to one flow-controlled stream per worker and adoption is likely abandoned -- [Cluster spike](/quest/m2/perf/kio-channel-contention/spike-cluster.md) - the same question with nothing shared diff --git a/quest/m2/perf/kio-channel-contention/spike-cluster.md b/quest/m2/perf/kio-channel-contention/spike-cluster.md deleted file mode 100644 index 6235cc431dc..00000000000 --- a/quest/m2/perf/kio-channel-contention/spike-cluster.md +++ /dev/null @@ -1,28 +0,0 @@ -# [M] Spike: relay per thread, clustered - -## Goal - -Measure a relay per worker thread, each with its own `Cluster` and origin and -nothing shared, clustered over the -in-memory transport (`moq_net::transport::memory`) instead -of QUIC. Compare it with the -[replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md), which -shares the control plane. The spike is a draft PR that never merges; its -numbers and a recommendation go in the -[epic](/quest/m2/perf/kio-channel-contention/README.md). - -## Plan - -- Cluster links dial only by URL over QUIC today - (`run_remote_session`, `rs/moq-relay/src/cluster.rs`); the spike adds a - throwaway seam to hand a cluster a pre-made memory session. - `moq_net::Client::connect` and `Server::accept` already take a `memory::Session`, - which meets their `Boxable` and `MaybeSync` bounds. -- Everything crosses workers through moq-lite: announcements, routes, and - groups. That isolates what sharing the control plane saves, and exercises - the cluster path the relay must keep working anyway. -- Flow control throttles only the hop, as in the replica spike. -- Same measurements and win criteria as the replica spike, plus announce propagation delay - between workers and the memory of W announce tables. -- A win goes to `/quest-plan` for a real implementation quest; a no-win - restores the perf epic's no-sharding rule. From 8ccf8e86052595771dc9c1bcf0d592bafec79ffe Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 14:05:30 -0800 Subject: [PATCH 5/5] quest: record the cluster spike verdict as a conditional win The maintainer settled the spike's decision on 2026-11-01: relay-per-thread is planned, on the condition that the worker mesh never re-announces peer routes to peers. Record the condition, the routing-semantics change, the unmeasured cost, and how it relates to the replica spike. Co-Authored-By: Claude Opus 5.5 --- .../m2/perf/kio-channel-contention/README.md | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/quest/m2/perf/kio-channel-contention/README.md b/quest/m2/perf/kio-channel-contention/README.md index 6bf6d23f3d0..091a34c8d7d 100644 --- a/quest/m2/perf/kio-channel-contention/README.md +++ b/quest/m2/perf/kio-channel-contention/README.md @@ -296,13 +296,34 @@ a full mesh re-announces every route to every peer (not checked route by route). At 16 workers a relay with 10,000 broadcasts would hold about 9 GB of announce state that a shared origin holds in 49 MB. -Verdict: the data-plane half is a win wherever a track fans out to many +Findings: the data-plane half is a win wherever a track fans out to many subscribers per worker (chat), and a wash or a loss where it does not -(fanout). Sharing nothing then costs W-squared announce state and -propagation, which grows past any data-plane saving at relay broadcast -counts. Relay-per-thread is not a win as a design; it bounds what the -[replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md), -which keeps one shared announce table, can save on the data plane. +(fanout). Sharing nothing through today's full mesh then costs W-squared +announce state and propagation, which grows past any data-plane saving at +relay broadcast counts. + +Decided (maintainer, 2026-11-01, cluster spike): + +- ✅ A conditional win. Relay-per-thread is planned through `/quest-plan`, + on the condition that the worker mesh never re-announces peer routes to + peers. Each worker then learns a broadcast once, from the worker that + ingested it, which cuts announce state to about one route per worker + instead of one per peer. +- That changes cluster routing semantics: today a relay re-announces every + route it learned to every other peer, and hop chains prevent loops. How + routes from outside the relay (other relays' peer links) reach every + worker without re-announce is for the planning to settle. +- The cost is unmeasured. The spike measured only the full mesh; nobody has + measured announce state, propagation, or the data plane with + no-re-announce. +- The [replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md) + (claimed in [moq-dev/moq#7533](https://github.com/moq-dev/moq/pull/7533), + queued) still runs. It keeps one shared announce table and replicates + only the data plane, so it chases the same data-plane win, which these + numbers bound, at a different control-plane cost. Whether + relay-per-thread waits on its result is for the planning to settle. +- Rejected: calling relay-per-thread a no-win and leaving sharding to the + replica spike alone, and restoring the no-sharding rule now. ## Required