From d0ab6c190fbd019e3bd2ecc6fd282315a95ed5a4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 09:56:38 -0700 Subject: [PATCH 1/4] Start quest/m1/session-burst-hang Co-Authored-By: GPT-6 From 9fbf37618bc5623ed1c72caa8a00a81fbeab5abd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 10:36:29 -0700 Subject: [PATCH 2/4] bench: provision lossless session burst working set Co-Authored-By: GPT-6 --- quest/m1/README.md | 1 - quest/m1/session-burst-hang.md | 17 ----------------- rs/moq-net/benches/session.rs | 30 +++++++++++++++++++++++++++++- 3 files changed, 29 insertions(+), 19 deletions(-) delete mode 100644 quest/m1/session-burst-hang.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 25c59174d55..b02b209548e 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -132,7 +132,6 @@ ladders. - [Parked-read bench budget](/quest/m1/parked-read-bench-budget.md) - `track_parked_read` completes at default settings instead of expiring its parked reads mid-warm-up - [Mergeable bench buckets](/quest/m1/bench-buckets.md) - moq-bench emits per-interval latency buckets that sum across processes and hosts - [Relay session bench](/quest/m1/bench-relay.md) - the same scenario through moq-relay's own connection handling -- [Session burst hang](/quest/m1/session-burst-hang.md) - the burst sweep completes at 16 subscriptions and 16+ groups per round - [Read-only lookup](/quest/m1/read-only-lookup.md) - a live track lookup stops waking the front and every demand watcher - [Generated @moq/net](/quest/m1/rs2ts/README.md) - the browser runs moq-net as TypeScript generated from the Rust source, retiring js/net's hand-written protocol and model code - [Audio jitter target](/quest/m1/audio-jitter-target/README.md) - the playout target is default in both languages; a manual browser proof and a native trace replay remain diff --git a/quest/m1/session-burst-hang.md b/quest/m1/session-burst-hang.md deleted file mode 100644 index bfda5fbc1ef..00000000000 --- a/quest/m1/session-burst-hang.md +++ /dev/null @@ -1,17 +0,0 @@ -# [S] The session burst bench completes at every shape - -## Goal - -`rs/moq-net/benches/session.rs`'s `burst` sweep runs to completion at 16 -subscriptions and 16 or more groups per round, with or without the serve -budgets, and the sweep covers those shapes again. - -## Plan - -Found while landing serve-budget -([#5088](https://github.com/moq-dev/moq/pull/5088)), which adds the `burst` -axis: with that axis applied, the bench hangs at those shapes without the -budget too, so the sweep was limited to shapes that complete. Decided 2026-10-08: -treat it as a possible session stall, not a bench setup limit. Find whether -it is cache size, stream credit, or a real stall, fix it at the cause, and -add a regression test if the cause is in the session rather than the bench. diff --git a/rs/moq-net/benches/session.rs b/rs/moq-net/benches/session.rs index 6078d923458..e64c53dbb2c 100644 --- a/rs/moq-net/benches/session.rs +++ b/rs/moq-net/benches/session.rs @@ -220,6 +220,7 @@ struct Cluster { _origins: Vec, _pairs: Vec, next_hop: u64, + pools: Vec, } impl Cluster { @@ -232,6 +233,7 @@ impl Cluster { _origins: Vec::new(), _pairs: Vec::new(), next_hop: 0, + pools: Vec::new(), }; for _ in 0..shape.relays { @@ -278,6 +280,7 @@ impl Cluster { self.next_hop += 1; let mut config = origin::Config::new(Hop::new(self.next_hop).unwrap()); config.pool = cache::Pool::new(cache::Config::default().with_capacity(capacity)); + self.pools.push(config.pool.clone()); let (producer, driver) = origin::Producer::new(config); support::harness::spawn(driver); producer @@ -337,6 +340,7 @@ struct Room { writing: Vec>, /// The group each viewer's subscriber is reading, when paced, flattened in viewer order. reading: Vec>, + cache_now: moq_net::time::Instant, } impl Room { @@ -368,6 +372,12 @@ impl Room { viewers.push(viewer); } + // First measure the complete batch without eviction. The delivery benchmark must + // fit its working set; cache-pressure loss is a different workload. + let capacities: Vec<_> = cluster.pools.iter().map(cache::Pool::capacity).collect(); + for pool in &cluster.pools { + pool.resize(None); + } let mut room = Self { cluster, viewers, @@ -382,6 +392,7 @@ impl Room { written: 0, writing: Vec::new(), reading: Vec::new(), + cache_now: moq_net::time::Instant::now(), }; room.writing.resize_with(room.watched.len(), || None); room.reading.resize_with(shape.viewers * shape.watch, || None); @@ -389,10 +400,27 @@ impl Room { for _ in 0..FRAMES / shape.frames() { room.round().await; } + for (pool, capacity) in room.cluster.pools.iter().zip(capacities) { + // Keep the old and current batch resident during the handover, including + // bookkeeping rather than estimating a group from payload bytes alone. + pool.resize(capacity.map(|capacity| capacity.max(pool.used() * 2))); + } + // Exercise turnover as well as the empty cache, including Criterion's one-pass + // nightly smoke mode. + for _ in 0..4 { + room.round().await; + } room } async fn round(&mut self) { + // Unpaced rounds otherwise collapse many complete batches into one cache + // tick, where yesterday's read outranks today's unread write. Sample a new + // recency epoch per batch without sleeping or changing protocol time. + self.cache_now += Duration::from_secs(1); + for pool in &self.cluster.pools { + pool.gc(self.cache_now); + } if self.shape.paced { return self.paced_round().await; } @@ -888,7 +916,7 @@ fn session(c: &mut Criterion) { delivery( c, "burst", - [(1, 1), (1, 8), (1, 64), (8, 1), (8, 8)].map(|(watch, burst)| Shape { + [(1, 1), (1, 8), (1, 64), (8, 1), (8, 8), (16, 16), (16, 64)].map(|(watch, burst)| Shape { publishers: 1, broadcasts: 16, viewers: 1, From 005ecd050da15e25fa550d8ecfc4aa128827aa08 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 10:37:45 -0700 Subject: [PATCH 3/4] quest: link completed session burst work Co-Authored-By: GPT-6 --- quest/m1/spinning-loops.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quest/m1/spinning-loops.md b/quest/m1/spinning-loops.md index a6ce372f850..3c2f0342dad 100644 --- a/quest/m1/spinning-loops.md +++ b/quest/m1/spinning-loops.md @@ -34,5 +34,5 @@ Decided 2026-10-09 while reworking #5088: ## Related -- [Session burst hang](/quest/m1/session-burst-hang.md) - the same bench shapes that hang +- [Session burst hang](https://github.com/moq-dev/moq/pull/5238) - the same bench shapes that hang - [FFI runtime](/quest/m1/ffi-runtime.md) - FFI apps run on more than one thread From 696a8f4e48873490e08d6973f004cb61ac9ddf9d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 11 Oct 2026 08:40:30 -0700 Subject: [PATCH 4/4] bench: tick session caches outside the timed round Codex flagged that every pool's gc ran inside Room::round, the timed body of each session_delivery bench, adding a pool-count-dependent cost to each sample. Measure now ticks the caches before timing each round. Viewer caches now live on the Viewer instead of Cluster.pools, so the join bench no longer retains every joined viewer's pool across iterations. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/benches/session.rs | 63 +++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/rs/moq-net/benches/session.rs b/rs/moq-net/benches/session.rs index e64c53dbb2c..916a3a9cf4d 100644 --- a/rs/moq-net/benches/session.rs +++ b/rs/moq-net/benches/session.rs @@ -220,6 +220,7 @@ struct Cluster { _origins: Vec, _pairs: Vec, next_hop: u64, + /// The relay and publisher caches; each viewer owns its own. pools: Vec, } @@ -237,8 +238,9 @@ impl Cluster { }; for _ in 0..shape.relays { - let relay = this.origin(RELAY_CACHE); + let (relay, pool) = this.origin(RELAY_CACHE); this.relays.push(relay); + this.pools.push(pool); } // Peer every pair the way a cluster dial does: each side publishes its @@ -257,7 +259,8 @@ impl Cluster { } for publisher in 0..shape.publishers { - let origin = this.origin(CLIENT_CACHE); + let (origin, pool) = this.origin(CLIENT_CACHE); + this.pools.push(pool); for index in 0..shape.broadcasts { let broadcast = origin .publish(path(publisher * shape.broadcasts + index), Default::default()) @@ -276,19 +279,19 @@ impl Cluster { this } - fn origin(&mut self, capacity: u64) -> origin::Producer { + fn origin(&mut self, capacity: u64) -> (origin::Producer, cache::Pool) { self.next_hop += 1; let mut config = origin::Config::new(Hop::new(self.next_hop).unwrap()); - config.pool = cache::Pool::new(cache::Config::default().with_capacity(capacity)); - self.pools.push(config.pool.clone()); + let pool = cache::Pool::new(cache::Config::default().with_capacity(capacity)); + config.pool = pool.clone(); let (producer, driver) = origin::Producer::new(config); support::harness::spawn(driver); - producer + (producer, pool) } /// Connect a viewer session to `relay`. async fn join(&mut self, relay: usize) -> Viewer { - let origin = self.origin(CLIENT_CACHE); + let (origin, pool) = self.origin(CLIENT_CACHE); let mut options = MockConnectOptions::new(self.version); // Hidden included, as the dash's `.stats/` scope sees its subtree. options.server_publish = Some(self.relays[relay].consume().with_hidden(true)); @@ -299,6 +302,7 @@ impl Cluster { subscribers: Vec::new(), _pair: pair, origin, + pool, } } } @@ -307,6 +311,7 @@ struct Viewer { subscribers: Vec, _pair: MockPair, origin: origin::Producer, + pool: cache::Pool, } impl Viewer { @@ -372,12 +377,6 @@ impl Room { viewers.push(viewer); } - // First measure the complete batch without eviction. The delivery benchmark must - // fit its working set; cache-pressure loss is a different workload. - let capacities: Vec<_> = cluster.pools.iter().map(cache::Pool::capacity).collect(); - for pool in &cluster.pools { - pool.resize(None); - } let mut room = Self { cluster, viewers, @@ -396,11 +395,19 @@ impl Room { }; room.writing.resize_with(room.watched.len(), || None); room.reading.resize_with(shape.viewers * shape.watch, || None); + + // First measure the complete batch without eviction. The delivery benchmark must + // fit its working set; cache-pressure loss is a different workload. + let capacities: Vec<_> = room.pools().map(cache::Pool::capacity).collect(); + for pool in room.pools() { + pool.resize(None); + } // Warm every path so the timed rounds skip first-group setup. for _ in 0..FRAMES / shape.frames() { + room.tick(); room.round().await; } - for (pool, capacity) in room.cluster.pools.iter().zip(capacities) { + for (pool, capacity) in room.pools().zip(capacities) { // Keep the old and current batch resident during the handover, including // bookkeeping rather than estimating a group from payload bytes alone. pool.resize(capacity.map(|capacity| capacity.max(pool.used() * 2))); @@ -408,19 +415,28 @@ impl Room { // Exercise turnover as well as the empty cache, including Criterion's one-pass // nightly smoke mode. for _ in 0..4 { + room.tick(); room.round().await; } room } - async fn round(&mut self) { - // Unpaced rounds otherwise collapse many complete batches into one cache - // tick, where yesterday's read outranks today's unread write. Sample a new - // recency epoch per batch without sleeping or changing protocol time. + fn pools(&self) -> impl Iterator { + let viewers = self.viewers.iter().map(|viewer| &viewer.pool); + self.cluster.pools.iter().chain(viewers) + } + + /// Unpaced rounds otherwise collapse many complete batches into one cache tick, + /// where yesterday's read outranks today's unread write. Sample a new recency + /// epoch per round without sleeping or changing protocol time. + fn tick(&mut self) { self.cache_now += Duration::from_secs(1); - for pool in &self.cluster.pools { + for pool in self.pools() { pool.gc(self.cache_now); } + } + + async fn round(&mut self) { if self.shape.paced { return self.paced_round().await; } @@ -472,12 +488,17 @@ impl Room { assert_eq!(bytes, self.shape.expected()); } + /// Time `iters` rounds, excluding each round's cache tick: its cost grows with the + /// number of pools, not with the delivery being measured. async fn measure(&mut self, iters: u64) -> Duration { - let start = Instant::now(); + let mut elapsed = Duration::ZERO; for _ in 0..iters { + self.tick(); + let start = Instant::now(); self.round().await; + elapsed += start.elapsed(); } - start.elapsed() + elapsed } }