From 565a0ca3fb13afeb92418a3d19a6c1647285b28e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:26:52 -0800 Subject: [PATCH 1/7] chore(quest): claim quest/m2/perf/kio-channel-contention/spike-replica.md Quest-Claim: 38669fd82df46a6e From 90eaf9917f107e572bea52198992dcb10691c364 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 02:05:26 -0800 Subject: [PATCH 2/7] spike(relay): per-worker data-plane replicas behind SPIKE_REPLICA Co-Authored-By: Claude Opus 5.5 --- bench/relay.sh | 27 +++++++++- rs/moq-relay/src/cluster.rs | 103 ++++++++++++++++++++++++++++++++++-- rs/moq-relay/src/relay.rs | 8 +++ 3 files changed, 132 insertions(+), 6 deletions(-) diff --git a/bench/relay.sh b/bench/relay.sh index 562c406d846..8673a89b2c3 100755 --- a/bench/relay.sh +++ b/bench/relay.sh @@ -152,7 +152,12 @@ start_relay() { log=$directory/relay-$attempt.log relay_config "$config" "$port" "$runtime" "$workers" - env "${environment[@]}" "$relay" "$config" >"$log" 2>&1 & + # SPIKE: BENCH_RELAY_CPUS pins the relay (taskset execs, so the PID stays the relay's). + if [[ -n ${BENCH_RELAY_CPUS:-} ]]; then + env "${environment[@]}" taskset -c "$BENCH_RELAY_CPUS" "$relay" "$config" >"$log" 2>&1 & + else + env "${environment[@]}" "$relay" "$config" >"$log" 2>&1 & + fi RELAY_PID=$! for _ in {1..100}; do @@ -283,7 +288,12 @@ run_workload() { local relay_pid=$RELAY_PID generator_pid='' finished url local profile=$directory/$PROFILE url=$(relay_url "$RELAY_PORT") - local -a load=("$LOAD_BIN" --file "$config" + local -a load=() + # SPIKE: BENCH_LOAD_CPUS pins the load generator away from the relay's cores. + if [[ -n ${BENCH_LOAD_CPUS:-} ]]; then + load=(taskset -c "$BENCH_LOAD_CPUS") + fi + load+=("$LOAD_BIN" --file "$config" --connect "$url" --name "bench-$label-$workload" --startup 2s --report 500ms --output "$stats" "${overrides[@]}") case $TRANSPORT in @@ -305,6 +315,15 @@ run_workload() { HOST_PID=$! # The second half of the host window, matching the summary's throughput window. # Its own process group, so cleanup also stops perf and its workload. + # SPIKE: BENCH_STAT counts the relay's instructions over the unprofiled run's + # steady window (the same 5 s as the summary's throughput and the profiles). + local stat_pid='' + if [[ -n ${BENCH_STAT:-} && -z $PROFILE ]]; then + (sleep 5 && exec perf stat -x, -o "$directory/stat.csv" \ + -e cpu_core/instructions/u,cpu_atom/instructions/u,cpu_core/instructions/k,cpu_atom/instructions/k,task-clock \ + -p "$relay_pid" -- sleep 5) >"$directory/stat.log" 2>&1 & + stat_pid=$! + fi case $PROFILE in lock | cpu) setsid python3 "$ROOT/bench/profile.py" "$PROFILE" "$relay_pid" "$profile" --delay 5 --duration 5 \ @@ -326,6 +345,10 @@ run_workload() { return 1 fi HOST_PID= + if [[ -n $stat_pid ]] && ! wait "$stat_pid"; then + cat "$directory/stat.log" >&2 + return 1 + fi if [[ -n $PROFILE ]]; then if ! wait "$PROFILE_PID"; then cat "$directory/profile.log" >&2 diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 71b1496b2e0..e2c25ed2e86 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -1151,6 +1151,12 @@ pub struct Cluster { /// (filtered by their auth token) and remote dials both read and write here. pub origin: origin::Producer, + /// SPIKE (replica): this worker's data-plane replica. Downstream sessions + /// subscribe from it instead of [`Self::origin`]; it is fed from the shared + /// origin over one in-memory moq-lite session per worker. `None` serves + /// subscribers from the shared origin, as today. + replica: Option, + /// Stats registry. One instance per relay; sessions pick a billing tier via /// [`stats::Registry::tier`](moq_net::stats::Registry::tier) at acceptance time /// (the default tier unless configured otherwise, or any label the auth API @@ -1274,6 +1280,7 @@ impl Cluster { connection_ids: Arc::default(), critical, origin, + replica: None, stats: moq_net::stats::Registry::disabled(), _stats_publisher: None, }) @@ -1449,7 +1456,64 @@ impl Cluster { /// equivalent per-request setter), which derives the read handle. On an /// upstream grant that handle never sees a route learned from another upstream. pub fn subscriber(&self, token: &auth::Token) -> Option { - self.mounted(token)?.scope(&token.root, &token.subscribe).ok() + let base = self.replica.as_ref().unwrap_or(&self.origin); + self.mounted_on(base, token)?.scope(&token.root, &token.subscribe).ok() + } + + /// SPIKE (replica): build this worker's replica on the current (worker) + /// runtime and return a cluster whose subscribers read from it. + /// + /// The replica is a second origin with its own hop, fed by a moq-lite session + /// over the in-memory transport: the server half publishes the shared origin + /// (hidden paths included), the client half lands every route in the replica. + /// A track subscribed on this worker is subscribed once over that session and + /// fanned out locally. Publishers still publish into the shared origin. The + /// returned sessions keep the hop alive. + pub(crate) async fn spike_replica(&self) -> anyhow::Result<(Self, [moq_net::Session; 2])> { + use moq_net::transport::memory; + + let mut config = self.origin.config(); + config.hop = Hop::random(); + let (replica, driver) = origin::Producer::new(config); + tokio::spawn(moq_net::time::run(driver)); + + // The newest moq-lite, fixed by ALPN as a cluster peer would negotiate it. + let version: moq_net::Version = "moq-lite-07-wip".parse().expect("known version"); + let mut transport = memory::Config::default(); + transport.protocol = Some(version.alpn().to_string()); + let (client_transport, server_transport) = memory::pair(transport); + let server = moq_net::Server::new() + .with_versions(version.into()) + .with_publisher(self.origin.clone().with_link(origin::Link::Peer).consume().with_hidden(true)) + .with_limits(self.peer_limits); + let client = moq_net::Client::new() + .with_versions(version.into()) + .with_subscriber(replica.clone().with_link(origin::Link::Peer)) + .with_limits(self.peer_limits); + let now = moq_net::time::Instant::now(); + // Each side's driver runs as soon as its half is up: the client's SETUP is + // sent by its driver, so the server's accept waits on it. + let accept = async { + let (session, driver) = server.accept(now, server_transport).await?; + tokio::spawn(moq_net::time::run(driver)); + Ok::<_, moq_net::Error>(session) + }; + let connect = async { + let (session, driver) = client.connect(now, client_transport).await?; + tokio::spawn(moq_net::time::run(driver)); + Ok::<_, moq_net::Error>(session) + }; + let (accepted, connected) = + tokio::time::timeout(std::time::Duration::from_secs(5), async { tokio::join!(accept, connect) }) + .await + .context("replica hop handshake timed out")?; + let server_session = accepted.context("replica hop accept")?; + let client_session = connected.context("replica hop connect")?; + tracing::info!(replica = %replica.hop(), shared = %self.origin.hop(), "spike: worker replica linked"); + + let mut cluster = self.clone(); + cluster.replica = Some(replica); + Ok((cluster, [server_session, client_session])) } /// Returns an [`origin::Producer`] scoped to this session's publish permissions, @@ -1464,10 +1528,14 @@ impl Cluster { /// before it is scoped to the session. A mount the origin refuses /// (overlapping another) admits nothing. fn mounted(&self, token: &auth::Token) -> Option { + self.mounted_on(&self.origin, token) + } + + fn mounted_on(&self, base: &origin::Producer, token: &auth::Token) -> Option { let mut origin = match (token.upstream, token.peer) { - (true, _) => self.origin.clone().with_link(origin::Link::Upstream), - (false, true) => self.origin.clone().with_link(origin::Link::Peer), - (false, false) => self.origin.clone(), + (true, _) => base.clone().with_link(origin::Link::Upstream), + (false, true) => base.clone().with_link(origin::Link::Peer), + (false, false) => base.clone(), }; for (at, target) in &token.mounts { let mounted = token.root.join(at).map_err(moq_net::Error::from); @@ -4036,4 +4104,31 @@ mod tests { None ); } + + /// SPIKE (replica): a broadcast published into the shared origin reaches a + /// subscriber of a worker's replica over the in-memory hop. + #[tokio::test] + async fn spike_replica_serves_the_shared_origin() { + let cluster = new_cluster(Config::default()).expect("cluster"); + let (worker, _hop) = cluster.spike_replica().await.expect("replica"); + let broadcast = cluster.origin.create_broadcast("cam").expect("create"); + let mut track = broadcast.create_track("data", timed()).expect("track"); + broadcast.announce(Default::default()).expect("announce"); + + let consumer = worker.replica.as_ref().expect("replica set").consume(); + let remote = kio::clock::timeout(Duration::from_secs(5), consumer.request("cam", None)) + .await + .expect("broadcast resolves") + .expect("broadcast present"); + let mut reader = remote.track("data").unwrap().subscribe(None).await.unwrap(); + let mut group = track.append_group().unwrap(); + group.write_frame(moq_net::Timestamp::ZERO, "hello").unwrap(); + group.finish().unwrap(); + let mut group = kio::clock::timeout(Duration::from_secs(5), reader.recv_group()) + .await + .expect("group arrives") + .unwrap() + .unwrap(); + assert_eq!(group.read_frame().await.unwrap().unwrap().payload, "hello".as_bytes()); + } } diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index 44a90d1e2d2..7799fb60473 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -846,6 +846,14 @@ async fn serve( // Each QUIC worker binds here; Relay::run binds the shared listener before // readiness and passes it to the same accept loop. let mut listener = server.listen().await.context("failed to bind listeners")?; + // SPIKE (replica): `SPIKE_REPLICA=1` gives each worker its own data-plane replica. + let (cluster, _hop) = match std::env::var_os("SPIKE_REPLICA") { + Some(_) => { + let (cluster, hop) = cluster.spike_replica().await?; + (cluster, Some(hop)) + } + None => (cluster, None), + }; serve_listening(&mut listener, cluster, auth, shutdown, sessions).await } From 0636821358e42363ba9b2a4f05e07f0f85ccb1a8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 02:48:48 -0800 Subject: [PATCH 3/7] spike(bench): 32 MiB perf buffer for the lock capture Co-Authored-By: Claude Opus 5.5 --- bench/profile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/profile.py b/bench/profile.py index c483a6d337f..0335f28eac7 100644 --- a/bench/profile.py +++ b/bench/profile.py @@ -175,9 +175,9 @@ def lock(pid, delay, duration, directory): # never sees a worker's switch-in; --switch-events records it. glibc's syscall wrapper # keeps no frame pointer, so frame-pointer unwinding drops its caller, lock_contended; # DWARF unwinding keeps it. Switch timestamps bound CPU and open waits alike. - # The default 512 KiB buffer drops samples at 16 workers; 2 MiB does not. + # The default 512 KiB buffer drops samples at 16 workers. SPIKE: 2 MiB still lost one on chat at 16 base workers, 8 MiB too, so 32 MiB. subprocess.run(['perf', 'record', '--quiet', '--no-buildid-cache', '-e', EVENT, '-c', '1', '--call-graph', 'dwarf', '--switch-events', - '-k', 'monotonic', '-m', '2M', '-p', str(pid), '-o', str(data), '--', 'sleep', str(duration)], check=True) + '-k', 'monotonic', '-m', '32M', '-p', str(pid), '-o', str(data), '--', 'sleep', str(duration)], check=True) after = schedstat(pid, names) result, folded = reduce(names, parse(perf_script(data, '--show-switch-events')), before, after) (directory / 'lock.json').write_text(json.dumps(result, indent=2) + '\n') From 62e637f329898b290d662011d5335c1f3cb3c060 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 03:02:46 -0800 Subject: [PATCH 4/7] spike: remove the replica spike code; it stays at 0636821358 Co-Authored-By: Claude Opus 5.5 --- bench/profile.py | 4 +- bench/relay.sh | 27 +--------- rs/moq-relay/src/cluster.rs | 103 ++---------------------------------- rs/moq-relay/src/relay.rs | 8 --- 4 files changed, 8 insertions(+), 134 deletions(-) diff --git a/bench/profile.py b/bench/profile.py index 0335f28eac7..c483a6d337f 100644 --- a/bench/profile.py +++ b/bench/profile.py @@ -175,9 +175,9 @@ def lock(pid, delay, duration, directory): # never sees a worker's switch-in; --switch-events records it. glibc's syscall wrapper # keeps no frame pointer, so frame-pointer unwinding drops its caller, lock_contended; # DWARF unwinding keeps it. Switch timestamps bound CPU and open waits alike. - # The default 512 KiB buffer drops samples at 16 workers. SPIKE: 2 MiB still lost one on chat at 16 base workers, 8 MiB too, so 32 MiB. + # The default 512 KiB buffer drops samples at 16 workers; 2 MiB does not. subprocess.run(['perf', 'record', '--quiet', '--no-buildid-cache', '-e', EVENT, '-c', '1', '--call-graph', 'dwarf', '--switch-events', - '-k', 'monotonic', '-m', '32M', '-p', str(pid), '-o', str(data), '--', 'sleep', str(duration)], check=True) + '-k', 'monotonic', '-m', '2M', '-p', str(pid), '-o', str(data), '--', 'sleep', str(duration)], check=True) after = schedstat(pid, names) result, folded = reduce(names, parse(perf_script(data, '--show-switch-events')), before, after) (directory / 'lock.json').write_text(json.dumps(result, indent=2) + '\n') diff --git a/bench/relay.sh b/bench/relay.sh index 8673a89b2c3..562c406d846 100755 --- a/bench/relay.sh +++ b/bench/relay.sh @@ -152,12 +152,7 @@ start_relay() { log=$directory/relay-$attempt.log relay_config "$config" "$port" "$runtime" "$workers" - # SPIKE: BENCH_RELAY_CPUS pins the relay (taskset execs, so the PID stays the relay's). - if [[ -n ${BENCH_RELAY_CPUS:-} ]]; then - env "${environment[@]}" taskset -c "$BENCH_RELAY_CPUS" "$relay" "$config" >"$log" 2>&1 & - else - env "${environment[@]}" "$relay" "$config" >"$log" 2>&1 & - fi + env "${environment[@]}" "$relay" "$config" >"$log" 2>&1 & RELAY_PID=$! for _ in {1..100}; do @@ -288,12 +283,7 @@ run_workload() { local relay_pid=$RELAY_PID generator_pid='' finished url local profile=$directory/$PROFILE url=$(relay_url "$RELAY_PORT") - local -a load=() - # SPIKE: BENCH_LOAD_CPUS pins the load generator away from the relay's cores. - if [[ -n ${BENCH_LOAD_CPUS:-} ]]; then - load=(taskset -c "$BENCH_LOAD_CPUS") - fi - load+=("$LOAD_BIN" --file "$config" + local -a load=("$LOAD_BIN" --file "$config" --connect "$url" --name "bench-$label-$workload" --startup 2s --report 500ms --output "$stats" "${overrides[@]}") case $TRANSPORT in @@ -315,15 +305,6 @@ run_workload() { HOST_PID=$! # The second half of the host window, matching the summary's throughput window. # Its own process group, so cleanup also stops perf and its workload. - # SPIKE: BENCH_STAT counts the relay's instructions over the unprofiled run's - # steady window (the same 5 s as the summary's throughput and the profiles). - local stat_pid='' - if [[ -n ${BENCH_STAT:-} && -z $PROFILE ]]; then - (sleep 5 && exec perf stat -x, -o "$directory/stat.csv" \ - -e cpu_core/instructions/u,cpu_atom/instructions/u,cpu_core/instructions/k,cpu_atom/instructions/k,task-clock \ - -p "$relay_pid" -- sleep 5) >"$directory/stat.log" 2>&1 & - stat_pid=$! - fi case $PROFILE in lock | cpu) setsid python3 "$ROOT/bench/profile.py" "$PROFILE" "$relay_pid" "$profile" --delay 5 --duration 5 \ @@ -345,10 +326,6 @@ run_workload() { return 1 fi HOST_PID= - if [[ -n $stat_pid ]] && ! wait "$stat_pid"; then - cat "$directory/stat.log" >&2 - return 1 - fi if [[ -n $PROFILE ]]; then if ! wait "$PROFILE_PID"; then cat "$directory/profile.log" >&2 diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index e2c25ed2e86..71b1496b2e0 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -1151,12 +1151,6 @@ pub struct Cluster { /// (filtered by their auth token) and remote dials both read and write here. pub origin: origin::Producer, - /// SPIKE (replica): this worker's data-plane replica. Downstream sessions - /// subscribe from it instead of [`Self::origin`]; it is fed from the shared - /// origin over one in-memory moq-lite session per worker. `None` serves - /// subscribers from the shared origin, as today. - replica: Option, - /// Stats registry. One instance per relay; sessions pick a billing tier via /// [`stats::Registry::tier`](moq_net::stats::Registry::tier) at acceptance time /// (the default tier unless configured otherwise, or any label the auth API @@ -1280,7 +1274,6 @@ impl Cluster { connection_ids: Arc::default(), critical, origin, - replica: None, stats: moq_net::stats::Registry::disabled(), _stats_publisher: None, }) @@ -1456,64 +1449,7 @@ impl Cluster { /// equivalent per-request setter), which derives the read handle. On an /// upstream grant that handle never sees a route learned from another upstream. pub fn subscriber(&self, token: &auth::Token) -> Option { - let base = self.replica.as_ref().unwrap_or(&self.origin); - self.mounted_on(base, token)?.scope(&token.root, &token.subscribe).ok() - } - - /// SPIKE (replica): build this worker's replica on the current (worker) - /// runtime and return a cluster whose subscribers read from it. - /// - /// The replica is a second origin with its own hop, fed by a moq-lite session - /// over the in-memory transport: the server half publishes the shared origin - /// (hidden paths included), the client half lands every route in the replica. - /// A track subscribed on this worker is subscribed once over that session and - /// fanned out locally. Publishers still publish into the shared origin. The - /// returned sessions keep the hop alive. - pub(crate) async fn spike_replica(&self) -> anyhow::Result<(Self, [moq_net::Session; 2])> { - use moq_net::transport::memory; - - let mut config = self.origin.config(); - config.hop = Hop::random(); - let (replica, driver) = origin::Producer::new(config); - tokio::spawn(moq_net::time::run(driver)); - - // The newest moq-lite, fixed by ALPN as a cluster peer would negotiate it. - let version: moq_net::Version = "moq-lite-07-wip".parse().expect("known version"); - let mut transport = memory::Config::default(); - transport.protocol = Some(version.alpn().to_string()); - let (client_transport, server_transport) = memory::pair(transport); - let server = moq_net::Server::new() - .with_versions(version.into()) - .with_publisher(self.origin.clone().with_link(origin::Link::Peer).consume().with_hidden(true)) - .with_limits(self.peer_limits); - let client = moq_net::Client::new() - .with_versions(version.into()) - .with_subscriber(replica.clone().with_link(origin::Link::Peer)) - .with_limits(self.peer_limits); - let now = moq_net::time::Instant::now(); - // Each side's driver runs as soon as its half is up: the client's SETUP is - // sent by its driver, so the server's accept waits on it. - let accept = async { - let (session, driver) = server.accept(now, server_transport).await?; - tokio::spawn(moq_net::time::run(driver)); - Ok::<_, moq_net::Error>(session) - }; - let connect = async { - let (session, driver) = client.connect(now, client_transport).await?; - tokio::spawn(moq_net::time::run(driver)); - Ok::<_, moq_net::Error>(session) - }; - let (accepted, connected) = - tokio::time::timeout(std::time::Duration::from_secs(5), async { tokio::join!(accept, connect) }) - .await - .context("replica hop handshake timed out")?; - let server_session = accepted.context("replica hop accept")?; - let client_session = connected.context("replica hop connect")?; - tracing::info!(replica = %replica.hop(), shared = %self.origin.hop(), "spike: worker replica linked"); - - let mut cluster = self.clone(); - cluster.replica = Some(replica); - Ok((cluster, [server_session, client_session])) + self.mounted(token)?.scope(&token.root, &token.subscribe).ok() } /// Returns an [`origin::Producer`] scoped to this session's publish permissions, @@ -1528,14 +1464,10 @@ impl Cluster { /// before it is scoped to the session. A mount the origin refuses /// (overlapping another) admits nothing. fn mounted(&self, token: &auth::Token) -> Option { - self.mounted_on(&self.origin, token) - } - - fn mounted_on(&self, base: &origin::Producer, token: &auth::Token) -> Option { let mut origin = match (token.upstream, token.peer) { - (true, _) => base.clone().with_link(origin::Link::Upstream), - (false, true) => base.clone().with_link(origin::Link::Peer), - (false, false) => base.clone(), + (true, _) => self.origin.clone().with_link(origin::Link::Upstream), + (false, true) => self.origin.clone().with_link(origin::Link::Peer), + (false, false) => self.origin.clone(), }; for (at, target) in &token.mounts { let mounted = token.root.join(at).map_err(moq_net::Error::from); @@ -4104,31 +4036,4 @@ mod tests { None ); } - - /// SPIKE (replica): a broadcast published into the shared origin reaches a - /// subscriber of a worker's replica over the in-memory hop. - #[tokio::test] - async fn spike_replica_serves_the_shared_origin() { - let cluster = new_cluster(Config::default()).expect("cluster"); - let (worker, _hop) = cluster.spike_replica().await.expect("replica"); - let broadcast = cluster.origin.create_broadcast("cam").expect("create"); - let mut track = broadcast.create_track("data", timed()).expect("track"); - broadcast.announce(Default::default()).expect("announce"); - - let consumer = worker.replica.as_ref().expect("replica set").consume(); - let remote = kio::clock::timeout(Duration::from_secs(5), consumer.request("cam", None)) - .await - .expect("broadcast resolves") - .expect("broadcast present"); - let mut reader = remote.track("data").unwrap().subscribe(None).await.unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(moq_net::Timestamp::ZERO, "hello").unwrap(); - group.finish().unwrap(); - let mut group = kio::clock::timeout(Duration::from_secs(5), reader.recv_group()) - .await - .expect("group arrives") - .unwrap() - .unwrap(); - assert_eq!(group.read_frame().await.unwrap().unwrap().payload, "hello".as_bytes()); - } } diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index 7799fb60473..44a90d1e2d2 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -846,14 +846,6 @@ async fn serve( // Each QUIC worker binds here; Relay::run binds the shared listener before // readiness and passes it to the same accept loop. let mut listener = server.listen().await.context("failed to bind listeners")?; - // SPIKE (replica): `SPIKE_REPLICA=1` gives each worker its own data-plane replica. - let (cluster, _hop) = match std::env::var_os("SPIKE_REPLICA") { - Some(_) => { - let (cluster, hop) = cluster.spike_replica().await?; - (cluster, Some(hop)) - } - None => (cluster, None), - }; serve_listening(&mut listener, cluster, auth, shutdown, sessions).await } From 69d91553e3c720ba381a3bf54cb121115ef2474b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 03:04:50 -0800 Subject: [PATCH 5/7] quest: report the replica spike, a conditional win Co-Authored-By: Claude Opus 5.5 --- .../m2/perf/kio-channel-contention/README.md | 82 ++++++++++++++++++- .../kio-channel-contention/log-adoption.md | 9 +- .../kio-channel-contention/spike-cluster.md | 6 +- .../kio-channel-contention/spike-replica.md | 39 --------- 4 files changed, 88 insertions(+), 48 deletions(-) delete mode 100644 quest/m2/perf/kio-channel-contention/spike-replica.md diff --git a/quest/m2/perf/kio-channel-contention/README.md b/quest/m2/perf/kio-channel-contention/README.md index ca1bdcac05e..b5a38435d68 100644 --- a/quest/m2/perf/kio-channel-contention/README.md +++ b/quest/m2/perf/kio-channel-contention/README.md @@ -140,8 +140,7 @@ Decided (maintainer, 2026-10-14, per-thread follow-up): the thread-per-core workers (`--runtime-workers`, the default once [Shard by default](/quest/m2/perf/shard-default.md) lands) and link workers over an in-memory transport (`moq_net::transport::memory`), - never real QUIC. The - [replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md) + never real QUIC. The replica spike (results below) replicates the data plane and shares announcements; the [cluster spike](/quest/m2/perf/kio-channel-contention/spike-cluster.md) shares nothing. A measured win reopens sharding through `/quest-plan`; a @@ -228,9 +227,86 @@ DWARF stacks, 36% incomplete) put waiter registration at 1.97% self-time, the fourth-largest site, on a 501-connection chat shape. It agrees with this epic's targets but carries no lock-wait split. +Replica spike (2026-11-01, +[moq-dev/moq#7533](https://github.com/moq-dev/moq/pull/7533)): the code is +`Cluster::spike_replica` in `rs/moq-relay/src/cluster.rs` at +[`0636821`](https://github.com/moq-dev/moq/blob/0636821358e42363ba9b2a4f05e07f0f85ccb1a8/rs/moq-relay/src/cluster.rs), +switched on by `SPIKE_REPLICA=1`, and did not merge. Each tokio worker +builds a second origin and links the shared one into it with one moq-lite-07 +session over `moq_net::transport::memory` (default 1 MiB stream buffer), +both halves on that worker. Publishers still publish into the shared origin; +the worker's subscribers read its replica. It departs from the plan twice: + +- Announcements cross the hop too, so each replica keeps its own announce + table. These shapes announce 1 to 256 broadcasts, so it doesn't show. +- Every subscription takes the hop, even on the publisher's own worker. So + the 1-worker rows are the hop's whole cost, and the 4-worker rows overstate + it by up to a quarter. + +Method: `just bench-profile lock`, tokio workers over QUIC, with `perf stat` +counting the relay's instructions over the unprofiled run's 5 s steady +window. The relay was pinned to the 16 E-cores and the generator to the 8 +P-cores. The harness pins everything to the P-cores by default, so these +numbers don't compare with the baseline above. Load average 5 to 21. Each +cell is the median of 3 runs (chat at 16 workers: 9 base, 5 replica; +video-heavy: 2). Instructions are user and kernel per delivered frame, in +thousands; CPU and RSS come from the host sampler; latency is the +generator's p50 in whole milliseconds. Each cell reads base → replica: + +| shape | workers | CPU cores | user instr | kernel instr | mutex wait (share) | RSS MiB | p50 ms | +| --- | --- | --- | --- | --- | --- | --- | --- | +| fanout | 1 | 0.06 → 0.05 | 167 → 170 | 20 → 21 | 0 → 0 | 36 → 36 | 3 → 4 | +| fanout | 4 | 0.15 → 0.06 | 170 → 177 | 52 → 30 | 39 ms (12%) → 9 ms (3.1%) | 37 → 38 | 3 → 3 | +| fanout | 16 | 0.45 → 0.16 | 182 → 211 | 344 → 123 | 807 ms (52%) → 256 ms (40%) | 43 → 47 | 3 → 2 | +| chat | 1 | 1.00 → 1.00 | 185 → 184 | 18 → 18 | 0 → 0 | 576 → 577 | 966 → 1,273 | +| chat | 4 | 1.91 → 1.80 | 168 → 171 | 40 → 20 | 50 ms (0.5%) → 18 ms (0.2%) | 499 → 507 | 31 → 68 | +| chat | 16 | 9.5 → 2.2 | 180 → 170 | 427 → 28 | not captured → 793 ms (7.9%) | 500 → 518 | 34 → 18 | +| video-heavy | 1 | 0.97 → 1.00 | 112 → 125 | 25 → 16 | 0 → 0 | 320 → 357 | 2.5 → 227 | +| video-heavy | 4 | 1.22 → 1.87 | 116 → 207 | 32 → 31 | 27 ms (0.4%) → 16 ms (0.2%) | 324 → 381 | 1 → 1 | +| video-heavy | 16 | 1.84 → 2.59 | 123 → 213 | 51 → 50 | 137 ms (1.5%) → 208 ms (1.6%) | 330 → 424 | 0 → 0 | + +`bench/workloads/video-heavy.toml` is the low-fan-out shape: 256 +connections each publish one 60 fps track of 60-frame groups and subscribe +to two others. Chat at 1 worker saturates the worker either way (about +15,000 of 20,000 messages/s delivered), so its latency is queueing. +Video-heavy at 1 worker saturates only with the replica: 28,400 instead of +31,400 frames/s. The harness refused all nine lock captures of base chat at +16 workers, at least seven of them for a lost perf record (2, 8, and 32 MiB +buffers), so its wait is missing. Its 400,000 +kernel instructions per frame are the contention: futex waits and wakes +around one track with 2,000 readers. Base video-heavy at 16 workers is one +capture; the other was refused for nonalternating switch records. + +- High fan-out wins at 16 workers. Chat drops from 9.5 to 2.2 cores, kernel + instructions per frame from 427k to 28k, and p50 from 34 to 18 ms. + Fanout drops from 0.45 to 0.16 cores while user instructions rise 16%, + which is the hop's encode and decode. At 4 workers the gain is small. +- What stays contended with replicas is mostly the shared track, whose + readers are now the hops, one per worker. In chat at 16 workers (four + captures), `TrackState` is 86% of the wait: `Subscriber::poll_drop` 28%, + `Subscriber::poll_finished` via `resume::Reader::learn_end` 26%, + `GroupExpiry::is_expired` 10%, and the rest 21%. Another 13% is the + per-track cache mutex in `cache::Track::watch_wall`, also under + `GroupExpiry::is_expired`. The version counter still targets those reads. +- Low fan-out loses. Video-heavy pays the hop on nearly every subscription: + 73 to 78% more user instructions per frame, 41 to 53% more CPU, and 12 to + 28% more memory, with no lock-wait gain at 16 workers. +- Memory: fanout and chat grow 0 to 11%. Replicated one-frame groups are + small, and multi-frame groups cost the most. + +Recommendation: a conditional win. Replicas pay off only on high-fan-out +tracks. Applied to every track, they cost low fan-out about 1.8 times the +user instructions per frame. If sharding reopens, replicate per track on demand: +never on the publisher's own worker, and only above a per-worker fan-out +threshold, serving the rest from the shared origin. Rerun this comparison +once the [Version counter](/quest/m2/perf/kio-channel-contention/version.md) +lands, since it attacks the same 16-worker chat waits without the hop. Weigh +it against the +[Cluster spike](/quest/m2/perf/kio-channel-contention/spike-cluster.md) +before `/quest-plan`. + ## Required - [Version counter](/quest/m2/perf/kio-channel-contention/version.md) - an opted-in re-poll skips the mutex when nothing changed, and parking no longer contends with readers -- [Replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md) - measures per-worker data-plane replicas linked over the in-memory transport, announcements shared - [Cluster spike](/quest/m2/perf/kio-channel-contention/spike-cluster.md) - measures a relay per thread clustered over the in-memory transport, nothing shared - [Log adoption](/quest/m2/perf/kio-channel-contention/log-adoption.md) - the rest of moq-net's log-shaped state moves onto the log where it measures as a win diff --git a/quest/m2/perf/kio-channel-contention/log-adoption.md b/quest/m2/perf/kio-channel-contention/log-adoption.md index afe533d180c..0f2e656724f 100644 --- a/quest/m2/perf/kio-channel-contention/log-adoption.md +++ b/quest/m2/perf/kio-channel-contention/log-adoption.md @@ -18,7 +18,11 @@ stays on its mutex. - Waits on the spikes (decided 2026-10-14 for the log, kept here): with per-thread sessions every consumer of a replica is on one thread, so its mutex is uncontended and a lock-free multi-reader log only pays off if the - shared model survives. + shared model survives. The replica spike (2026-11-01, in the + [epic](/quest/m2/perf/kio-channel-contention/README.md)) does not settle + it: replicas lose on low fan-out, so shared tracks would stay, and the + shared track it keeps is still the most contended channel, with one reader + per worker. - Audit moq-net (and moq-relay's `critical.rs`) for other state that is appended at the back, read by cursor, and evicted from the front, beyond `GroupState` frames, which the log quest already moves. List the @@ -29,5 +33,4 @@ stays on its mutex. ## Required -- [Replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md) - if per-worker replicas win, each channel's cross-core readers collapse to one flow-controlled stream per worker and adoption is likely abandoned -- [Cluster spike](/quest/m2/perf/kio-channel-contention/spike-cluster.md) - the same question with nothing shared +- [Cluster spike](/quest/m2/perf/kio-channel-contention/spike-cluster.md) - if a relay per thread wins, each channel's cross-core readers collapse to one flow-controlled stream per worker and adoption is likely abandoned diff --git a/quest/m2/perf/kio-channel-contention/spike-cluster.md b/quest/m2/perf/kio-channel-contention/spike-cluster.md index 6235cc431dc..384c500c372 100644 --- a/quest/m2/perf/kio-channel-contention/spike-cluster.md +++ b/quest/m2/perf/kio-channel-contention/spike-cluster.md @@ -5,9 +5,9 @@ Measure a relay per worker thread, each with its own `Cluster` and origin and nothing shared, clustered over the in-memory transport (`moq_net::transport::memory`) instead -of QUIC. Compare it with the -[replica spike](/quest/m2/perf/kio-channel-contention/spike-replica.md), which -shares the control plane. The spike is a draft PR that never merges; its +of QUIC. Compare it with the replica spike, which shares the control plane; +its numbers are in the +[epic](/quest/m2/perf/kio-channel-contention/README.md). The spike is a draft PR that never merges; its numbers and a recommendation go in the [epic](/quest/m2/perf/kio-channel-contention/README.md). diff --git a/quest/m2/perf/kio-channel-contention/spike-replica.md b/quest/m2/perf/kio-channel-contention/spike-replica.md deleted file mode 100644 index 2bd5933c621..00000000000 --- a/quest/m2/perf/kio-channel-contention/spike-replica.md +++ /dev/null @@ -1,39 +0,0 @@ -# [M] Spike: per-worker replicas - -## Goal - -Measure whether per-worker data-plane replicas cut lock wait on the -thread-per-core relay (`--runtime-workers`). Each worker serves tracks and -groups from its own origin, fed once per worker over the -in-memory transport (`moq_net::transport::memory`), while -announcements and routes stay in one shared origin. The spike is a draft PR -that never merges; its numbers and a recommendation go in the -[epic](/quest/m2/perf/kio-channel-contention/README.md). - -## Plan - -- Today every worker's sessions share one `origin::Producer`, built in - `Cluster::new` (`rs/moq-relay/src/cluster.rs`) and cloned per worker - (`rs/moq-relay/src/relay.rs`), so a subscriber on worker B reads state - written on worker A. -- With replicas, a track subscribed on worker B is subscribed once over a - memory session to the worker that holds it, then fanned out locally. Every - consumer of a replica is on one thread, so its kio mutex is uncontended, and - cross-core readers per frame drop from the subscriber count to the worker - count. -- Control plane shared, data plane replicated: the shared origin still - carries announcements, so announce churn is not paid once per worker. -- Flow control throttles only the hop (decided 2026-10-14): a slow worker's - stream lags and its stale groups reset; the publisher and other workers - never wait. -- Measure against the [epic](/quest/m2/perf/kio-channel-contention/README.md)'s - baseline: `just bench-profile lock` fanout and chat at 1, 4, and 16 workers, plus - relay CPU, resident memory (replicated group state), and glass-to-glass - latency added by the hop. Lower lock wait alone is not a win; weigh it - against the CPU, memory, and latency. Chat (`bench/workloads/chat.toml`) - is one broadcast to 2,000 viewers with one-frame groups, so it stresses - per-group track writes rather than frame fan-out. Replicas give little to - low-fanout load; if that matters, name a separate low-fanout workload - instead of reading it from chat. -- A win goes to `/quest-plan` for a real implementation quest; a no-win - restores the perf epic's no-sharding rule. From 638b2e30801fda80a53be7b189cdf64b05ddc7cf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 03:06:03 -0800 Subject: [PATCH 6/7] quest: don't attribute the hop's user instructions past what was measured Co-Authored-By: Claude Opus 5.5 --- quest/m2/perf/kio-channel-contention/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quest/m2/perf/kio-channel-contention/README.md b/quest/m2/perf/kio-channel-contention/README.md index b5a38435d68..8b2797a7801 100644 --- a/quest/m2/perf/kio-channel-contention/README.md +++ b/quest/m2/perf/kio-channel-contention/README.md @@ -280,7 +280,7 @@ capture; the other was refused for nonalternating switch records. - High fan-out wins at 16 workers. Chat drops from 9.5 to 2.2 cores, kernel instructions per frame from 427k to 28k, and p50 from 34 to 18 ms. Fanout drops from 0.45 to 0.16 cores while user instructions rise 16%, - which is the hop's encode and decode. At 4 workers the gain is small. + the hop's own session work. At 4 workers the gain is small. - What stays contended with replicas is mostly the shared track, whose readers are now the hops, one per worker. In chat at 16 workers (four captures), `TrackState` is 86% of the wait: `Subscriber::poll_drop` 28%, From 318d8d97f24fb626812899922f2234ea389e8f75 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 09:38:30 -0800 Subject: [PATCH 7/7] quest: scope the replica spike's lock-wait medians, video-heavy ranges, and version-counter claim Co-Authored-By: Claude Opus 5.5 --- .../m2/perf/kio-channel-contention/README.md | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/quest/m2/perf/kio-channel-contention/README.md b/quest/m2/perf/kio-channel-contention/README.md index 8b2797a7801..7dd49b13e1b 100644 --- a/quest/m2/perf/kio-channel-contention/README.md +++ b/quest/m2/perf/kio-channel-contention/README.md @@ -207,10 +207,12 @@ median): deque 30, runs 30, log 46, ring 42. Draining it: 14, 15, 12, 19. growing stops every reader or needs epochs. Its per-slot locks cost 40% more for a burst on one thread and lose to the log across cores. It wins one-frame groups only because its reader holds no channel handle. -- Not run: `just bench-profile lock` on a multi-frame shape +- Not run on this host: `just bench-profile lock` on a multi-frame shape (`bench/workloads/video-heavy.toml` is one), which needs - `kernel.perf_event_paranoid=-1` and this host has 4. So a relay's share of - `GroupState` wait with multi-frame groups is still unmeasured. + `kernel.perf_event_paranoid=-1` and this host has 4. The replica spike + below captured video-heavy's total lock wait but doesn't split it by + channel, so a relay's share of `GroupState` wait with multi-frame groups + is still unmeasured. Decided (maintainer, 2026-10-24): Front eviction lands on the lock-free append-only log (`kio::log`) itself, a tail run from @@ -249,8 +251,9 @@ window. The relay was pinned to the 16 E-cores and the generator to the 8 P-cores. The harness pins everything to the P-cores by default, so these numbers don't compare with the baseline above. Load average 5 to 21. Each cell is the median of 3 runs (chat at 16 workers: 9 base, 5 replica; -video-heavy: 2). Instructions are user and kernel per delivered frame, in -thousands; CPU and RSS come from the host sampler; latency is the +video-heavy: 2). Mutex wait is the median of the runs whose lock capture +the harness accepted, which is fewer where noted below. Instructions are +user and kernel per delivered frame, in thousands; CPU and RSS come from the host sampler; latency is the generator's p50 in whole milliseconds. Each cell reads base → replica: | shape | workers | CPU cores | user instr | kernel instr | mutex wait (share) | RSS MiB | p50 ms | @@ -287,21 +290,26 @@ capture; the other was refused for nonalternating switch records. `Subscriber::poll_finished` via `resume::Reader::learn_end` 26%, `GroupExpiry::is_expired` 10%, and the rest 21%. Another 13% is the per-track cache mutex in `cache::Track::watch_wall`, also under - `GroupExpiry::is_expired`. The version counter still targets those reads. -- Low fan-out loses. Video-heavy pays the hop on nearly every subscription: - 73 to 78% more user instructions per frame, 41 to 53% more CPU, and 12 to - 28% more memory, with no lock-wait gain at 16 workers. + `GroupExpiry::is_expired`. The version counter targets the `TrackState` + reads, and only the ones that find nothing changed: a read of new data + still locks, and the cache mutex isn't a channel, so it stays. +- Low fan-out loses. Video-heavy pays the hop on nearly every subscription. + At 4 and 16 workers that is 73 to 78% more user instructions per frame, + 41 to 53% more CPU, and 18 to 28% more memory, with no lock-wait gain at + 16 workers. At 1 worker the hop saturates the worker instead: 12% more + user instructions and 12% more memory, CPU pinned at one core, fewer + frames delivered, and p50 from 2.5 to 227 ms. - Memory: fanout and chat grow 0 to 11%. Replicated one-frame groups are small, and multi-frame groups cost the most. Recommendation: a conditional win. Replicas pay off only on high-fan-out tracks. Applied to every track, they cost low fan-out about 1.8 times the -user instructions per frame. If sharding reopens, replicate per track on demand: -never on the publisher's own worker, and only above a per-worker fan-out +user instructions per frame at 4 and 16 workers. If sharding reopens, +replicate per track on demand: never on the publisher's own worker, and only above a per-worker fan-out threshold, serving the rest from the shared origin. Rerun this comparison once the [Version counter](/quest/m2/perf/kio-channel-contention/version.md) -lands, since it attacks the same 16-worker chat waits without the hop. Weigh -it against the +lands, since it attacks the same 16-worker chat `TrackState` waits without +the hop. Weigh it against the [Cluster spike](/quest/m2/perf/kio-channel-contention/spike-cluster.md) before `/quest-plan`.