From d195c4c4c94d4aa0957425d3551ba59285bbec88 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 06:29:50 -0800 Subject: [PATCH 01/16] chore(quest): claim quest/m1/release-perf/relay-cpu-bisect.md Quest-Claim: fb8c6036951f2c28 From 92b877834f5a221449858bf9acee1702049b8d39 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 08:10:52 -0800 Subject: [PATCH 02/16] perf(moq-net): walk the live groups through one branch, not a chain `TrackState::live` chained two flattened options, so every step of every search for live content checked both, and `find_servable` scanned the single-run case through a `dyn` iterator. One enum branch per call and a static scan cut the chat fan-out by 891 instructions per delivered group through the in-process session (110,014 to 109,123) and 270 through the model (9,637 to 9,367), at 512 subscribers. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/track.rs | 57 ++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index e198ac52c3e..37b9e00e40c 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1458,16 +1458,16 @@ impl TrackState { } /// The live feed's cached groups within `range`, by sequence: never fetched backfill. - fn live(&self, range: impl ops::RangeBounds + Clone) -> impl DoubleEndedIterator { + fn live(&self, range: impl ops::RangeBounds) -> impl DoubleEndedIterator { // Without backfill every cached group is the feed's, so the cache is walked as is. - let all = self.visible.len() == self.lookup.len(); - let cached = all.then(|| self.lookup.range(range.clone())); - let indexed = (!all).then(|| { - self.visible - .range(range) - .map(|sequence| (sequence, self.lookup.get(sequence).expect("an indexed live slot"))) - }); - cached.into_iter().flatten().chain(indexed.into_iter().flatten()) + match self.visible.len() == self.lookup.len() { + true => LiveWalk::Cached(self.lookup.range(range)), + false => LiveWalk::Indexed( + self.visible + .range(range) + .map(|sequence| (sequence, self.lookup.get(sequence).expect("an indexed live slot"))), + ), + } } /// The first live-feed group at or above `from` that `servable` covers for which `f` @@ -1481,10 +1481,6 @@ impl TrackState { rev: bool, mut f: impl FnMut(&'a Slot) -> Option, ) -> Option { - let mut scan = |range: &mut dyn DoubleEndedIterator| match rev { - true => range.rev().find_map(|(_, slot)| f(slot)), - false => range.map(|(_, slot)| slot).find_map(&mut f), - }; match servable { Servable::Run { start, end } => { let from = from.max(*start); @@ -1493,7 +1489,11 @@ impl TrackState { return None; } let end = end.map_or(Bound::Unbounded, Bound::Excluded); - scan(&mut self.live((Bound::Included(from), end))) + let mut live = self.live((Bound::Included(from), end)); + match rev { + true => live.rev().find_map(|(_, slot)| f(slot)), + false => live.find_map(|(_, slot)| f(slot)), + } } Servable::Runs(several) => { let hi = match several.cap { @@ -6113,6 +6113,35 @@ impl Published { } } +/// [`TrackState::live`]'s walk, one branch per call rather than a chain checking both on +/// every step: every search for live content goes through it. +enum LiveWalk { + /// The cache, which holds only the live feed. + Cached(C), + /// The live index, each sequence looked up in the cache. + Indexed(I), +} + +impl, I: Iterator> Iterator for LiveWalk { + type Item = T; + + fn next(&mut self) -> Option { + match self { + Self::Cached(cached) => cached.next(), + Self::Indexed(indexed) => indexed.next(), + } + } +} + +impl, I: DoubleEndedIterator> DoubleEndedIterator for LiveWalk { + fn next_back(&mut self) -> Option { + match self { + Self::Cached(cached) => cached.next_back(), + Self::Indexed(indexed) => indexed.next_back(), + } + } +} + /// What one run's lookup found, for [`TrackState::find_servable`] leapfrogging the cache. enum Leap { /// A group in the run that the search answered. From c18d11f6f1d81561c97c52b2068a2230f70b36f8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 08:55:46 -0800 Subject: [PATCH 03/16] docs(bench): attribute the ranged reads' session cost on main Co-Authored-By: Claude Opus 5.5 --- .github/relay-cpu-bisect.json | 257 ++++++++++++++++++++++++++++++++++ .github/relay-cpu-bisect.md | 25 +++- 2 files changed, 280 insertions(+), 2 deletions(-) diff --git a/.github/relay-cpu-bisect.json b/.github/relay-cpu-bisect.json index b97fc71bf25..9140393add7 100644 --- a/.github/relay-cpu-bisect.json +++ b/.github/relay-cpu-bisect.json @@ -5111,6 +5111,263 @@ "per_group": 113142.93781887754 } ] + }, + { + "artifact": "nixos2:run.4rAIYS", + "note": "moq-dev/moq#6995 and moq-dev/moq#7542 each against its parent, beside main at fa9c1bf2a7 (current), with callgrind", + "harness_revision": "fa9c1bf2a713e5cddf1ec9600a831662308f77a0", + "frame_pointers": false, + "counts": [ + { + "label": "base-3f5939eb2ee3", + "revision": "3f5939eb2ee39f4ff9d52f6c061ff366110ced6a", + "scenario": "model", + "subscribers": 64, + "per_group": 11383.62037 + }, + { + "label": "base-3f5939eb2ee3", + "revision": "3f5939eb2ee39f4ff9d52f6c061ff366110ced6a", + "scenario": "model", + "subscribers": 512, + "per_group": 11183.88592810421 + }, + { + "label": "base-81fe8a6631cf", + "revision": "81fe8a6631cf4c3402ab7e04a1dbf3ccb61ba264", + "scenario": "model", + "subscribers": 64, + "per_group": 11946.947874 + }, + { + "label": "base-81fe8a6631cf", + "revision": "81fe8a6631cf4c3402ab7e04a1dbf3ccb61ba264", + "scenario": "model", + "subscribers": 512, + "per_group": 11741.044139225627 + }, + { + "label": "base-4eb9c23db5de", + "revision": "4eb9c23db5de2baa48ef838c357392fa6424fa2a", + "scenario": "model", + "subscribers": 64, + "per_group": 12130.777 + }, + { + "label": "base-4eb9c23db5de", + "revision": "4eb9c23db5de2baa48ef838c357392fa6424fa2a", + "scenario": "model", + "subscribers": 512, + "per_group": 11924.981775164726 + }, + { + "label": "base-f1034b7cc973", + "revision": "f1034b7cc973009bb33abfd06404be1a1dea0266", + "scenario": "model", + "subscribers": 64, + "per_group": 13030.641598 + }, + { + "label": "base-f1034b7cc973", + "revision": "f1034b7cc973009bb33abfd06404be1a1dea0266", + "scenario": "model", + "subscribers": 512, + "per_group": 12824.633319272965 + }, + { + "label": "current", + "revision": "fa9c1bf2a713e5cddf1ec9600a831662308f77a0", + "scenario": "model", + "subscribers": 64, + "per_group": 9842.698621 + }, + { + "label": "current", + "revision": "fa9c1bf2a713e5cddf1ec9600a831662308f77a0", + "scenario": "model", + "subscribers": 512, + "per_group": 9636.831055687053 + }, + { + "label": "base-3f5939eb2ee3", + "revision": "3f5939eb2ee39f4ff9d52f6c061ff366110ced6a", + "scenario": "session", + "subscribers": 64, + "per_group": 142667.32649552144 + }, + { + "label": "base-3f5939eb2ee3", + "revision": "3f5939eb2ee39f4ff9d52f6c061ff366110ced6a", + "scenario": "session", + "subscribers": 512, + "per_group": 140201.8951291454 + }, + { + "label": "base-81fe8a6631cf", + "revision": "81fe8a6631cf4c3402ab7e04a1dbf3ccb61ba264", + "scenario": "session", + "subscribers": 64, + "per_group": 146173.30892114525 + }, + { + "label": "base-81fe8a6631cf", + "revision": "81fe8a6631cf4c3402ab7e04a1dbf3ccb61ba264", + "scenario": "session", + "subscribers": 512, + "per_group": 143490.09075055804 + }, + { + "label": "base-4eb9c23db5de", + "revision": "4eb9c23db5de2baa48ef838c357392fa6424fa2a", + "scenario": "session", + "subscribers": 64, + "per_group": 118886.59577935061 + }, + { + "label": "base-4eb9c23db5de", + "revision": "4eb9c23db5de2baa48ef838c357392fa6424fa2a", + "scenario": "session", + "subscribers": 512, + "per_group": 116810.04079639669 + }, + { + "label": "base-f1034b7cc973", + "revision": "f1034b7cc973009bb33abfd06404be1a1dea0266", + "scenario": "session", + "subscribers": 64, + "per_group": 121965.10142754318 + }, + { + "label": "base-f1034b7cc973", + "revision": "f1034b7cc973009bb33abfd06404be1a1dea0266", + "scenario": "session", + "subscribers": 512, + "per_group": 119858.4774593431 + }, + { + "label": "current", + "revision": "fa9c1bf2a713e5cddf1ec9600a831662308f77a0", + "scenario": "session", + "subscribers": 64, + "per_group": 111825.13812579974 + }, + { + "label": "current", + "revision": "fa9c1bf2a713e5cddf1ec9600a831662308f77a0", + "scenario": "session", + "subscribers": 512, + "per_group": 110023.01396085777 + } + ], + "callgrind_session_512": { + "unit": "instructions per delivered group, from bench/fanout.py functions and compare", + "moq-dev/moq#6995 merge against parent, self": { + "TrackState::live": 2042.1, + "Chain> in live": 738.7, + "btree Range::next_back": 340.1, + "BTreeMap::range, removed": -1724.3, + "Cursor::probe": 1115.0, + "TrackState::drift_edge": -328.3, + "program total": 2974.5 + }, + "moq-dev/moq#7542 merge against parent": { + "runs::Layered::first inclusive": 2921.7, + "runs::Layered::last inclusive": 887.7, + "program total": 3042.7 + }, + "inclusive by revision": { + "TrackState::live": { + "81fe8a6631": 3782.1, + "f1034b7cc9": 2816.9, + "fa9c1bf2a7": 2839.6 + }, + "runs::Layered::first": { + "f1034b7cc9": 2921.7, + "fa9c1bf2a7": 1515.4 + }, + "runs::Layered::last": { + "f1034b7cc9": 887.7, + "fa9c1bf2a7": 601.7 + }, + "Cursor::probe": { + "81fe8a6631": 19258.3, + "f1034b7cc9": 16174.8, + "fa9c1bf2a7": 9399.6 + } + }, + "self by revision": { + "Cursor::probe": { + "81fe8a6631": 4343.2, + "f1034b7cc9": 3226.0, + "fa9c1bf2a7": 1842.4 + } + }, + "TrackState::live callers on fa9c1bf2a7": "drift_edge 18.2M and arrival 6.7M of 24.9M inclusive instructions over the run" + } + }, + { + "artifact": "nixos2:run.m71Z8E", + "note": "this PR's live walk (92b877834f, counted as a working-tree patch on fa9c1bf2a7, so both labels record fa9c1bf2a7) against main at fa9c1bf2a7 (base)", + "harness_revision": "fa9c1bf2a713e5cddf1ec9600a831662308f77a0", + "frame_pointers": false, + "counts": [ + { + "label": "base", + "revision": "fa9c1bf2a713e5cddf1ec9600a831662308f77a0", + "scenario": "model", + "subscribers": 64, + "per_group": 9842.699317 + }, + { + "label": "base", + "revision": "fa9c1bf2a713e5cddf1ec9600a831662308f77a0", + "scenario": "model", + "subscribers": 512, + "per_group": 9636.831806350754 + }, + { + "label": "current", + "revision": "fa9c1bf2a713e5cddf1ec9600a831662308f77a0", + "scenario": "model", + "subscribers": 64, + "per_group": 9572.745755 + }, + { + "label": "current", + "revision": "fa9c1bf2a713e5cddf1ec9600a831662308f77a0", + "scenario": "model", + "subscribers": 512, + "per_group": 9366.837443825167 + }, + { + "label": "base", + "revision": "fa9c1bf2a713e5cddf1ec9600a831662308f77a0", + "scenario": "session", + "subscribers": 64, + "per_group": 111823.2947856686 + }, + { + "label": "base", + "revision": "fa9c1bf2a713e5cddf1ec9600a831662308f77a0", + "scenario": "session", + "subscribers": 512, + "per_group": 110014.30896245217 + }, + { + "label": "current", + "revision": "fa9c1bf2a713e5cddf1ec9600a831662308f77a0", + "scenario": "session", + "subscribers": 64, + "per_group": 110923.76357565579 + }, + { + "label": "current", + "revision": "fa9c1bf2a713e5cddf1ec9600a831662308f77a0", + "scenario": "session", + "subscribers": 512, + "per_group": 109123.37050581952 + } + ] } ], "quiet_host_runs": [ diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md index 12a1a385969..3b5bfba0cf0 100644 --- a/.github/relay-cpu-bisect.md +++ b/.github/relay-cpu-bisect.md @@ -94,7 +94,7 @@ In-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and The session column starts at the 2026-11-01 harness; blank cells predate it. [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995)'s session cost is the probe's walk of the live groups (`TrackState::live`, +4.0k inclusive) on both the reader's cursor and its forward-drop cursor. -## Fixes in this PR +## Fixes in moq-dev/moq#7605 Each is its own commit, counted in one run (run.s6IjbN in the evidence) at 512 subscribers per delivered group, each row on top of the one above: @@ -103,7 +103,7 @@ Each is its own commit, counted in one run (run.s6IjbN in the evidence) at 512 s | `main` | 13,300 | 120,948 | | A live reader's probe skips the fills while it misses nothing | 10,477 (-2,823) | 114,139 (-6,809) | | A window with one set of runs reads without growing across two | 10,126 (-351) | 113,133 (-1,007) | -| This branch | 10,126 (-23.9%) | 113,133 (-6.5%) | +| All of them | 10,126 (-23.9%) | 113,133 (-6.5%) | Against `main` at [619df6e1b3](https://github.com/moq-dev/moq/commit/619df6e1b3) (run.evL1wx), after merging it: model 13,368 to 10,119 (-24.3%), session 121,114 to 113,143 (-6.6%). @@ -113,6 +113,27 @@ kio's `WaiterList::register` is the session path's second-largest self cost: 7.2 On the quiet test host (run.we4umM, one-minute load at most 4.7), chat over io\_uring went from 154.5k \[154.1-155.0] instructions per frame on `main` to 148.9k \[148.3-150.8] with the probe and window changes (-3.6%), four rounds each, every cell at 20,000 frames/s with no loss. The relay's QUIC, crypto, and syscalls are most of its cost, so the model's -23% shows as a few percent there. +## The ranged reads' session cost on `main` + +[moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995) and [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542) each added about 3k instructions per group to the session when they landed. Run 4rAIYS counts each against its parent again, beside `main` at [fa9c1bf2a7](https://github.com/moq-dev/moq/commit/fa9c1bf2a7), with callgrind attribution (512 subscribers, per delivered group): + +| Change | Parent | Merge | Change | +| --- | ---: | ---: | ---: | +| Fill linearly in range count and span, [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995) | 140,202 | 143,490 | +3,288 | +| Layered windows, [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542) | 116,810 | 119,859 | +3,049 | + +Neither cost is in the probe's judgement, so the fast path left both in place: + +- **[moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995), the live index's walk.** It indexed the live feed's groups apart from fetched backfill, so every search for live content (`TrackState::live`) walked a chain of two flattened options where it had walked the cache's range. The walk's callers are the drift edge each read measures (`poll_drift` → `drift_edge` → `live_edge`) and the arrival walk, on the reader's cursor and its forward-drop cursor alike; the judgement never calls it. It still cost 2,840 inclusive per group on `main`, as at [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542)'s merge (2,817). The probe's own share of the PR (+1,115 self) did fall with the fast path: the probe's self cost went from 4,343 at the merge to 1,842 on `main`. +- **[moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542), layered window queries.** A window answers each query from its shared fixed runs and a layer of moving ones (`runs::Layered::first` and `last`): +3,810 inclusive at the merge, less about 800 in the range-map queries they replaced. [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)'s single-set fast path brought them to 2,117 on `main`, about 40 queries per delivery at about 50 instructions each, from the forward-drop walk, peeks, admission, and the probe. That is the number of window queries a per-reader window answers, not a walk, so it is accepted in the [release-perf epic](../quest/m1/release-perf/README.md). + +`TrackState::live` now walks either the cache or the live index through one branch per call, and the drift edge's single-run search scans it statically rather than through a `dyn` iterator (run.m71Z8E, against `main` at [fa9c1bf2a7](https://github.com/moq-dev/moq/commit/fa9c1bf2a7)): + +| Scenario | `main` | Live walk | Change | +| --- | ---: | ---: | ---: | +| Model | 9,637 | 9,367 | -270 (-2.8%) | +| Session | 110,014 | 109,123 | -891 (-0.8%) | + ## Validation Every count, delivery estimate, loss figure, and host load cited here is in [the evidence](relay-cpu-bisect.json), keyed by run artifact. Raw captures (`perf.data`, folded stacks, reports) stay in the measuring checkout's ignored `.tmp/bench/`. From 428ce1f836fa2a41b5db71da3989eacbf40d246f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 09:54:08 -0800 Subject: [PATCH 04/16] fix(bench): keep each revision's checkout so a later run reuses its build A fresh worktree per run gave every source a new mtime, so on a host without a compiler cache each run rebuilt every workspace crate of every revision, though its target persisted. Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 4 ++-- bench/run.sh | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/bench/README.md b/bench/README.md index 3df19a59dbf..65b540ca1cd 100644 --- a/bench/README.md +++ b/bench/README.md @@ -328,8 +328,8 @@ The relay and generator keep these affinities for every revision. Instructions mode runs alone. Each base builds into its own target under `target/bench/revisions/`, -from a detached worktree in `.tmp/` that the run removes when it exits, so a -later run reuses that build. The revisions build side by side on every allowed +from a detached worktree at `.tmp/bench-revisions/` that later runs keep +using, so a later run reuses that build without a compiler cache. The revisions build side by side on every allowed CPU, since fat LTO links on one thread, `BENCH_BUILD_JOBS` (default 4) at a time: each link takes gigabytes, so lower it on a host with little memory. Every round measures each base and the diff --git a/bench/run.sh b/bench/run.sh index 356c7282b8a..655ea1e719d 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -973,15 +973,19 @@ run_queued_builds() { done } -# Check out one revision for this run, in the checkout's ignored scratch space, -# where it loads only its own Cargo configuration. Its target persists, keyed by -# revision; the compiler cache keys on remapped paths, so a fresh checkout reuses it. +# Check out one revision in the checkout's ignored scratch space, where it loads +# only its own Cargo configuration. Both the checkout and its target persist, +# keyed by revision: a fresh checkout's new mtimes would rebuild every workspace +# crate on a host without a compiler cache. revision_checkout() { - local sha=$1 label=$2 worktree - mkdir -p "$ROOT/.tmp" - worktree=$(mktemp -d "$ROOT/.tmp/moq-bench-base.XXXXXX") - WORKTREES+=("$worktree") - git worktree add --detach "$worktree" "$sha" >/dev/null + local sha=$1 label=$2 worktree=$ROOT/.tmp/bench-revisions/$1 + if [[ ! -f $worktree/.git || $(git -C "$worktree" rev-parse HEAD 2>/dev/null) != "$sha" ]] || + [[ -n $(git -C "$worktree" status --porcelain 2>/dev/null) ]]; then + git worktree remove --force "$worktree" >/dev/null 2>&1 || rm -rf "$worktree" + git worktree prune + mkdir -p "$ROOT/.tmp/bench-revisions" + git worktree add --detach "$worktree" "$sha" >/dev/null + fi mkdir -p "$RUN/base-context/$label" (cd "$worktree" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context/$label" "$sha" --profile) REVISION_CHECKOUT=$worktree From df10b0b79d5a2728a72ff1efdebe315e05728da0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 12:20:54 -0800 Subject: [PATCH 05/16] docs(quest): accept the ranged reads' remaining session cost Co-Authored-By: Claude Opus 5.5 --- quest/m1/release-perf/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/quest/m1/release-perf/README.md b/quest/m1/release-perf/README.md index ffabf4712d9..3fe376bf309 100644 --- a/quest/m1/release-perf/README.md +++ b/quest/m1/release-perf/README.md @@ -312,6 +312,24 @@ The first four were counted with frame pointers forced and the last two without, so compare each only with its own parent. None grows with subscribers. +Re-measured on `main` with the probe fast path, as that decision asked, and +accepted under it (see the +[report](/.github/relay-cpu-bisect.md#the-ranged-reads-session-cost-on-main)), +per delivered group at 512 subscribers through the in-process session: + +- Filling linearly in range count and span + ([moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995)): +3,288 + when it landed, mostly the live index's walk, which the drift edge reads on + every poll. The fast path never reached it; walking it through one branch + took back 891 (session) and 270 (model). What remains looks up the live + groups apart from fetched backfill, which keeps backfill out of every + search for live content. +- Layered windows + ([moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542)): +3,049 + when it landed, the window queries `runs::Layered::first` and `last`. They + cost 2,117 on `main`, about 40 queries per delivery at about 50 + instructions each: per-reader windows answered by search, not a walk. + ## Required - [Relay CPU per frame since #5926](/quest/m1/release-perf/relay-cpu-bisect.md) - the 18-56% rise in relay instructions per delivered frame is bisected, attributed, and its avoidable cost fixed From 35c904d8463aa93fae64dd81fe1a3e584a265d1d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 17:02:04 -0800 Subject: [PATCH 06/16] docs(bench): attribute qmux, receive leases, and public ingest on the relay; re-pair main Co-Authored-By: Claude Opus 5.5 --- .github/relay-cpu-bisect.json | 1949 +++++++++++++++++++++ .github/relay-cpu-bisect.md | 54 +- quest/m1/release-perf/README.md | 7 + quest/m1/release-perf/relay-cpu-bisect.md | 48 +- 4 files changed, 2007 insertions(+), 51 deletions(-) diff --git a/.github/relay-cpu-bisect.json b/.github/relay-cpu-bisect.json index 9140393add7..ab7430b569c 100644 --- a/.github/relay-cpu-bisect.json +++ b/.github/relay-cpu-bisect.json @@ -5492,6 +5492,1955 @@ } } } + }, + { + "artifact": "nixos2:run.nPd1Nb", + "host": "nixos2 (i7-8700; no quest gate there, so each run started after 80 s at load under 2.5 with no other process above half a core)", + "relay_cpus": "0-3", + "generator_cpus": "4,5,10,11", + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "workload": "chat", + "runtime": "io-uring-workers", + "revisions": { + "base-cdff3c8f952b": "cdff3c8f95, the old endpoint", + "base-674624b74972": "674624b749, before moq-dev/moq#6231 (in-tree qmux)", + "base-72259ab3cc3d": "72259ab3cc, moq-dev/moq#6231 merged", + "base-43fc9251d71b": "43fc9251d7, before moq-dev/moq#5878 (receive leases)", + "base-756637200cb1": "756637200c, moq-dev/moq#5878 merged", + "current": "fa9c1bf2a7 (main)" + }, + "note": "in-tree qmux and receive leases against their parents, with the old endpoint and main; superseded by run.S0G1Z7 and run.ssZrgC: another benchmark ran on the host from 10:32 to 10:41 PST (round 2) and other agents loaded it from 10:53 on (rounds 3-4)", + "summary": { + "base-cdff3c8f952b": { + "4": { + "cells": [ + { + "cell": "relay/base-cdff3c8f952b/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 107293.44947647768, + "cycles_per_frame": 256614.11540890482, + "recv_fps": 19999.99641179648, + "groups_lost": 0, + "host_load_1m": 3.41 + }, + { + "cell": "relay/base-cdff3c8f952b/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 112020.88567258869, + "cycles_per_frame": 233427.64383328473, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 2.03 + }, + { + "cell": "relay/base-cdff3c8f952b/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 104193.32625231904, + "cycles_per_frame": 349500.36762604164, + "recv_fps": 22070.137181124872, + "groups_lost": 10495, + "host_load_1m": 18.63 + }, + { + "cell": "relay/base-cdff3c8f952b/round-4/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 102647.33240158105, + "cycles_per_frame": 335409.25206338876, + "recv_fps": 23331.915662838535, + "groups_lost": 0, + "host_load_1m": 11.97 + } + ], + "instructions_per_frame": { + "median": 105743.38786439836, + "min": 102647.33240158105, + "max": 112020.88567258869 + }, + "cycles_per_frame": { + "median": 296011.6837361468, + "min": 233427.64383328473, + "max": 349500.36762604164 + }, + "recv_fps": { + "median": 21035.068590562434, + "min": 19999.99641179648, + "max": 23331.915662838535 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 10495 + }, + "host_load_1m": { + "median": 7.69, + "min": 2.03, + "max": 18.63 + } + } + }, + "base-674624b74972": { + "4": { + "cells": [ + { + "cell": "relay/base-674624b74972/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 116064.0182945456, + "cycles_per_frame": 284038.0372225478, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 4.27 + }, + { + "cell": "relay/base-674624b74972/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 119225.8016337427, + "cycles_per_frame": 278378.5001455593, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 2.91 + }, + { + "cell": "relay/base-674624b74972/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 108407.79553900972, + "cycles_per_frame": 390530.5129091481, + "recv_fps": 22679.533831544733, + "groups_lost": 3670, + "host_load_1m": 13.79 + }, + { + "cell": "relay/base-674624b74972/round-4/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 109920.84379254498, + "cycles_per_frame": 380286.23060173256, + "recv_fps": 20879.050875971, + "groups_lost": 0, + "host_load_1m": 12.59 + } + ], + "instructions_per_frame": { + "median": 112992.4310435453, + "min": 108407.79553900972, + "max": 119225.8016337427 + }, + "cycles_per_frame": { + "median": 332162.13391214015, + "min": 278378.5001455593, + "max": 390530.5129091481 + }, + "recv_fps": { + "median": 20439.5254379855, + "min": 20000.0, + "max": 22679.533831544733 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 3670 + }, + "host_load_1m": { + "median": 8.43, + "min": 2.91, + "max": 13.79 + } + } + }, + "base-72259ab3cc3d": { + "4": { + "cells": [ + { + "cell": "relay/base-72259ab3cc3d/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 117491.92032608019, + "cycles_per_frame": 267471.6747536516, + "recv_fps": 19996.000391367954, + "groups_lost": 0, + "host_load_1m": 3.15 + }, + { + "cell": "relay/base-72259ab3cc3d/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 118893.42468247692, + "cycles_per_frame": 305343.3812316238, + "recv_fps": 20001.037748663013, + "groups_lost": 0, + "host_load_1m": 3.22 + }, + { + "cell": "relay/base-72259ab3cc3d/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 120580.73424893965, + "cycles_per_frame": 261145.4873614282, + "recv_fps": 19998.4049150517, + "groups_lost": 0, + "host_load_1m": 2.14 + }, + { + "cell": "relay/base-72259ab3cc3d/round-4/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 97343.83487083412, + "cycles_per_frame": 307231.7547016757, + "recv_fps": 23739.94315358436, + "groups_lost": 34379, + "host_load_1m": 12.18 + } + ], + "instructions_per_frame": { + "median": 118192.67250427855, + "min": 97343.83487083412, + "max": 120580.73424893965 + }, + "cycles_per_frame": { + "median": 286407.5279926377, + "min": 261145.4873614282, + "max": 307231.7547016757 + }, + "recv_fps": { + "median": 19999.721331857356, + "min": 19996.000391367954, + "max": 23739.94315358436 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 34379 + }, + "host_load_1m": { + "median": 3.185, + "min": 2.14, + "max": 12.18 + } + } + }, + "base-43fc9251d71b": { + "4": { + "cells": [ + { + "cell": "relay/base-43fc9251d71b/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 125822.8702265522, + "cycles_per_frame": 271575.53130708146, + "recv_fps": 19997.580153327697, + "groups_lost": 0, + "host_load_1m": 2.37 + }, + { + "cell": "relay/base-43fc9251d71b/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 126226.75918792057, + "cycles_per_frame": 265659.0141503531, + "recv_fps": 20001.5564720836, + "groups_lost": 0, + "host_load_1m": 3.28 + }, + { + "cell": "relay/base-43fc9251d71b/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 132963.85918654423, + "cycles_per_frame": 298522.45293065876, + "recv_fps": 20003.999636556462, + "groups_lost": 0, + "host_load_1m": 1.69 + }, + { + "cell": "relay/base-43fc9251d71b/round-4/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 125943.03006772001, + "cycles_per_frame": 392626.6292770254, + "recv_fps": 20082.981629599166, + "groups_lost": 0, + "host_load_1m": 12.54 + } + ], + "instructions_per_frame": { + "median": 126084.8946278203, + "min": 125822.8702265522, + "max": 132963.85918654423 + }, + "cycles_per_frame": { + "median": 285048.99211887014, + "min": 265659.0141503531, + "max": 392626.6292770254 + }, + "recv_fps": { + "median": 20002.77805432003, + "min": 19997.580153327697, + "max": 20082.981629599166 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 2.825, + "min": 1.69, + "max": 12.54 + } + } + }, + "base-756637200cb1": { + "4": { + "cells": [ + { + "cell": "relay/base-756637200cb1/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 131695.32839197942, + "cycles_per_frame": 315275.38207840413, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 2.45 + }, + { + "cell": "relay/base-756637200cb1/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 128990.19482318428, + "cycles_per_frame": 296849.98342244996, + "recv_fps": 20001.562462420403, + "groups_lost": 0, + "host_load_1m": 2.62 + }, + { + "cell": "relay/base-756637200cb1/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 129729.59919803806, + "cycles_per_frame": 355960.2182700421, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 3.31 + }, + { + "cell": "relay/base-756637200cb1/round-4/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 103470.06997544979, + "cycles_per_frame": 368195.424479838, + "recv_fps": 20927.985147646843, + "groups_lost": 3519, + "host_load_1m": 15.46 + } + ], + "instructions_per_frame": { + "median": 129359.89701061117, + "min": 103470.06997544979, + "max": 131695.32839197942 + }, + "cycles_per_frame": { + "median": 335617.80017422314, + "min": 296849.98342244996, + "max": 368195.424479838 + }, + "recv_fps": { + "median": 20000.781231210203, + "min": 20000.0, + "max": 20927.985147646843 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 3519 + }, + "host_load_1m": { + "median": 2.965, + "min": 2.45, + "max": 15.46 + } + } + }, + "current": { + "4": { + "cells": [ + { + "cell": "relay/current/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 135638.3369038264, + "cycles_per_frame": 281050.02885593346, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 3.22 + }, + { + "cell": "relay/current/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 138452.5155594443, + "cycles_per_frame": 402598.4455126273, + "recv_fps": 19900.352110393003, + "groups_lost": 0, + "host_load_1m": 8.88 + }, + { + "cell": "relay/current/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 139559.37530375927, + "cycles_per_frame": 433917.9115404439, + "recv_fps": 19044.24484081281, + "groups_lost": 0, + "host_load_1m": 5.25 + }, + { + "cell": "relay/current/round-4/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 136821.02481219388, + "cycles_per_frame": 411649.08287637756, + "recv_fps": 23207.15104141366, + "groups_lost": 0, + "host_load_1m": 11.45 + } + ], + "instructions_per_frame": { + "median": 137636.77018581907, + "min": 135638.3369038264, + "max": 139559.37530375927 + }, + "cycles_per_frame": { + "median": 407123.76419450244, + "min": 281050.02885593346, + "max": 433917.9115404439 + }, + "recv_fps": { + "median": 19950.1760551965, + "min": 19044.24484081281, + "max": 23207.15104141366 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 7.065, + "min": 3.22, + "max": 11.45 + } + } + } + } + }, + { + "artifact": "nixos2:run.ssZrgC", + "host": "nixos2 (i7-8700; no quest gate there, so each run started after 80 s at load under 2.5 with no other process above half a core)", + "relay_cpus": "0-3", + "generator_cpus": "4,5,10,11", + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "workload": "chat", + "runtime": "io-uring-workers", + "revisions": { + "base-674624b74972": "674624b749, before moq-dev/moq#6231 (in-tree qmux)", + "base-72259ab3cc3d": "72259ab3cc, moq-dev/moq#6231 merged", + "base-43fc9251d71b": "43fc9251d7, before moq-dev/moq#5878 (receive leases)", + "base-756637200cb1": "756637200c, moq-dev/moq#5878 merged", + "current": "fa9c1bf2a7 (main)" + }, + "note": "in-tree qmux and receive leases, each against its parent, three quiet rounds", + "summary": { + "base-674624b74972": { + "4": { + "cells": [ + { + "cell": "relay/base-674624b74972/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 122951.71995644919, + "cycles_per_frame": 255971.83943255042, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 3.22 + }, + { + "cell": "relay/base-674624b74972/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 120152.74519201617, + "cycles_per_frame": 247535.1134764069, + "recv_fps": 20000.90810255991, + "groups_lost": 0, + "host_load_1m": 1.65 + }, + { + "cell": "relay/base-674624b74972/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 121268.01209849074, + "cycles_per_frame": 252832.79017265842, + "recv_fps": 20000.004874391467, + "groups_lost": 0, + "host_load_1m": 1.98 + } + ], + "instructions_per_frame": { + "median": 121268.01209849074, + "min": 120152.74519201617, + "max": 122951.71995644919 + }, + "cycles_per_frame": { + "median": 252832.79017265842, + "min": 247535.1134764069, + "max": 255971.83943255042 + }, + "recv_fps": { + "median": 20000.004874391467, + "min": 20000.0, + "max": 20000.90810255991 + }, + "groups_lost": { + "median": 0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 1.98, + "min": 1.65, + "max": 3.22 + } + } + }, + "base-72259ab3cc3d": { + "4": { + "cells": [ + { + "cell": "relay/base-72259ab3cc3d/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 116487.73864539976, + "cycles_per_frame": 322745.19617305894, + "recv_fps": 20005.242525393518, + "groups_lost": 0, + "host_load_1m": 2.95 + }, + { + "cell": "relay/base-72259ab3cc3d/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 119442.7442376012, + "cycles_per_frame": 290409.16399039346, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 2.8 + }, + { + "cell": "relay/base-72259ab3cc3d/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 122598.85023234437, + "cycles_per_frame": 247270.55289301067, + "recv_fps": 19937.584989648178, + "groups_lost": 0, + "host_load_1m": 2.13 + } + ], + "instructions_per_frame": { + "median": 119442.7442376012, + "min": 116487.73864539976, + "max": 122598.85023234437 + }, + "cycles_per_frame": { + "median": 290409.16399039346, + "min": 247270.55289301067, + "max": 322745.19617305894 + }, + "recv_fps": { + "median": 20000.0, + "min": 19937.584989648178, + "max": 20005.242525393518 + }, + "groups_lost": { + "median": 0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 2.8, + "min": 2.13, + "max": 2.95 + } + } + }, + "base-43fc9251d71b": { + "4": { + "cells": [ + { + "cell": "relay/base-43fc9251d71b/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 134383.72124410505, + "cycles_per_frame": 270687.509726875, + "recv_fps": 20000.787516353554, + "groups_lost": 0, + "host_load_1m": 2.26 + }, + { + "cell": "relay/base-43fc9251d71b/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 130907.65981918285, + "cycles_per_frame": 338466.9148330873, + "recv_fps": 20001.509273304517, + "groups_lost": 0, + "host_load_1m": 3.53 + }, + { + "cell": "relay/base-43fc9251d71b/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 134442.35488086086, + "cycles_per_frame": 261526.35964515782, + "recv_fps": 20000.004522207015, + "groups_lost": 0, + "host_load_1m": 1.97 + } + ], + "instructions_per_frame": { + "median": 134383.72124410505, + "min": 130907.65981918285, + "max": 134442.35488086086 + }, + "cycles_per_frame": { + "median": 270687.509726875, + "min": 261526.35964515782, + "max": 338466.9148330873 + }, + "recv_fps": { + "median": 20000.787516353554, + "min": 20000.004522207015, + "max": 20001.509273304517 + }, + "groups_lost": { + "median": 0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 2.26, + "min": 1.97, + "max": 3.53 + } + } + }, + "base-756637200cb1": { + "4": { + "cells": [ + { + "cell": "relay/base-756637200cb1/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 135052.80333709394, + "cycles_per_frame": 304416.9337026374, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 2.58 + }, + { + "cell": "relay/base-756637200cb1/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 135159.83535621187, + "cycles_per_frame": 296107.5328275039, + "recv_fps": 19960.534692361925, + "groups_lost": 0, + "host_load_1m": 2.13 + }, + { + "cell": "relay/base-756637200cb1/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 136499.2375161751, + "cycles_per_frame": 274499.05530350225, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 2.24 + } + ], + "instructions_per_frame": { + "median": 135159.83535621187, + "min": 135052.80333709394, + "max": 136499.2375161751 + }, + "cycles_per_frame": { + "median": 296107.5328275039, + "min": 274499.05530350225, + "max": 304416.9337026374 + }, + "recv_fps": { + "median": 20000.0, + "min": 19960.534692361925, + "max": 20000.0 + }, + "groups_lost": { + "median": 0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 2.24, + "min": 2.13, + "max": 2.58 + } + } + }, + "current": { + "4": { + "cells": [ + { + "cell": "relay/current/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 141565.8413293819, + "cycles_per_frame": 332402.78685216536, + "recv_fps": 19998.532326335226, + "groups_lost": 0, + "host_load_1m": 2.11 + }, + { + "cell": "relay/current/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 143133.34262850747, + "cycles_per_frame": 290122.6365970305, + "recv_fps": 20000.003363867825, + "groups_lost": 0, + "host_load_1m": 2.44 + }, + { + "cell": "relay/current/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 142331.72457972387, + "cycles_per_frame": 302572.7012703506, + "recv_fps": 20001.267371543225, + "groups_lost": 0, + "host_load_1m": 1.83 + } + ], + "instructions_per_frame": { + "median": 142331.72457972387, + "min": 141565.8413293819, + "max": 143133.34262850747 + }, + "cycles_per_frame": { + "median": 302572.7012703506, + "min": 290122.6365970305, + "max": 332402.78685216536 + }, + "recv_fps": { + "median": 20000.003363867825, + "min": 19998.532326335226, + "max": 20001.267371543225 + }, + "groups_lost": { + "median": 0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 2.11, + "min": 1.83, + "max": 2.44 + } + } + } + } + }, + { + "artifact": "nixos2:run.YxLYgw", + "host": "nixos2 (i7-8700; no quest gate there, so each run started after 80 s at load under 2.5 with no other process above half a core)", + "relay_cpus": "0-3", + "generator_cpus": "4,5,10,11", + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "workload": "chat", + "runtime": "io-uring-workers", + "revisions": { + "base-72259ab3cc3d": "72259ab3cc, moq-dev/moq#6231 merged", + "base-d44813cb703b": "d44813cb70, before moq-dev/moq#6142 (signed priorities)", + "base-b4dfafac8879": "b4dfafac88, moq-dev/moq#6142 merged", + "base-1f5caa7026bc": "1f5caa7026, moq-dev/moq#6279 merged (GSO batch end)", + "base-28a8c9a7a665": "28a8c9a7a6, before moq-dev/moq#6111 (public ingest)", + "base-9f9149812ce3": "9f9149812c, moq-dev/moq#6111 merged", + "base-a9a85580a448": "a9a85580a4, moq-dev/moq#6108 merged (m0 simplification audit)", + "base-43fc9251d71b": "43fc9251d7, before moq-dev/moq#5878 (receive leases)", + "current": "fa9c1bf2a7 (main)" + }, + "note": "every candidate between the qmux merge and the leases parent: the step is public ingest (moq-dev/moq#6111), in every round", + "attribution": { + "unit": "instructions per frame, base-28a8c9a7a665 against base-9f9149812ce3 in rounds 2 and 3, each function's share of the fixed-period stacks times the cell's count", + "inclusive": { + "kio Tasks::poll (each viewer's group serving)": 5333, + "group::Consumer::poll_read_frame": 2645, + "group::Consumer::poll_next_frame (new)": 1931, + "poll_expired_if_blocked (new)": 1993, + "group::Consumer::poll_expired_while_pending": 1931, + "resume::Recover::poll_budget": 1158, + "expiry::Wakes::watch_landing": 892, + "track state lock poll in Cursor::poll_drop (new closure)": 2020, + "track state lock poll in poll_recv_with (new closure)": 1035, + "track state lock poll in poll_stale (new closure)": 756 + }, + "self": { + "unlock": 1264, + "lock": 763, + "record": 762, + "memmove": 516 + } + }, + "summary": { + "base-72259ab3cc3d": { + "4": { + "cells": [ + { + "cell": "relay/base-72259ab3cc3d/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 116600.80403378875, + "cycles_per_frame": 296404.0251797434, + "recv_fps": 19996.917291659734, + "groups_lost": 0, + "host_load_1m": 7.36 + }, + { + "cell": "relay/base-72259ab3cc3d/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 120678.54920229728, + "cycles_per_frame": 259910.03051973105, + "recv_fps": 20001.739928956864, + "groups_lost": 0, + "host_load_1m": 12.25 + }, + { + "cell": "relay/base-72259ab3cc3d/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 119530.10853187667, + "cycles_per_frame": 274015.0971549953, + "recv_fps": 19997.146480161893, + "groups_lost": 0, + "host_load_1m": 3.26 + } + ], + "instructions_per_frame": { + "median": 119530.10853187667, + "min": 116600.80403378875, + "max": 120678.54920229728 + }, + "cycles_per_frame": { + "median": 274015.0971549953, + "min": 259910.03051973105, + "max": 296404.0251797434 + }, + "recv_fps": { + "median": 19997.146480161893, + "min": 19996.917291659734, + "max": 20001.739928956864 + }, + "groups_lost": { + "median": 0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 7.36, + "min": 3.26, + "max": 12.25 + } + } + }, + "base-d44813cb703b": { + "4": { + "cells": [ + { + "cell": "relay/base-d44813cb703b/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 118467.2804576051, + "cycles_per_frame": 303606.9456271673, + "recv_fps": 19799.4798960785, + "groups_lost": 0, + "host_load_1m": 4.71 + }, + { + "cell": "relay/base-d44813cb703b/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 116177.8865474754, + "cycles_per_frame": 335911.7712660491, + "recv_fps": 20003.159301608357, + "groups_lost": 0, + "host_load_1m": 7.59 + }, + { + "cell": "relay/base-d44813cb703b/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 122443.11178381351, + "cycles_per_frame": 259874.8655057333, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 2.13 + } + ], + "instructions_per_frame": { + "median": 118467.2804576051, + "min": 116177.8865474754, + "max": 122443.11178381351 + }, + "cycles_per_frame": { + "median": 303606.9456271673, + "min": 259874.8655057333, + "max": 335911.7712660491 + }, + "recv_fps": { + "median": 20000.0, + "min": 19799.4798960785, + "max": 20003.159301608357 + }, + "groups_lost": { + "median": 0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 4.71, + "min": 2.13, + "max": 7.59 + } + } + }, + "base-b4dfafac8879": { + "4": { + "cells": [ + { + "cell": "relay/base-b4dfafac8879/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 117706.50888977521, + "cycles_per_frame": 295017.2280609578, + "recv_fps": 20000.39539981791, + "groups_lost": 0, + "host_load_1m": 4.58 + }, + { + "cell": "relay/base-b4dfafac8879/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 111386.88587200461, + "cycles_per_frame": 366468.7396328849, + "recv_fps": 23357.823035955247, + "groups_lost": 0, + "host_load_1m": 15.53 + }, + { + "cell": "relay/base-b4dfafac8879/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 121470.82141956154, + "cycles_per_frame": 262853.85043358454, + "recv_fps": 19998.819524420833, + "groups_lost": 0, + "host_load_1m": 4.47 + } + ], + "instructions_per_frame": { + "median": 117706.50888977521, + "min": 111386.88587200461, + "max": 121470.82141956154 + }, + "cycles_per_frame": { + "median": 295017.2280609578, + "min": 262853.85043358454, + "max": 366468.7396328849 + }, + "recv_fps": { + "median": 20000.39539981791, + "min": 19998.819524420833, + "max": 23357.823035955247 + }, + "groups_lost": { + "median": 0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 4.58, + "min": 4.47, + "max": 15.53 + } + } + }, + "base-1f5caa7026bc": { + "4": { + "cells": [ + { + "cell": "relay/base-1f5caa7026bc/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 122293.53437754198, + "cycles_per_frame": 259402.52604331865, + "recv_fps": 20002.39666995295, + "groups_lost": 0, + "host_load_1m": 2.73 + }, + { + "cell": "relay/base-1f5caa7026bc/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 114063.49945336058, + "cycles_per_frame": 373406.21364514437, + "recv_fps": 22082.540562045557, + "groups_lost": 0, + "host_load_1m": 11.77 + }, + { + "cell": "relay/base-1f5caa7026bc/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 118722.77636613933, + "cycles_per_frame": 282982.7753007605, + "recv_fps": 20003.99958246546, + "groups_lost": 0, + "host_load_1m": 2.7 + } + ], + "instructions_per_frame": { + "median": 118722.77636613933, + "min": 114063.49945336058, + "max": 122293.53437754198 + }, + "cycles_per_frame": { + "median": 282982.7753007605, + "min": 259402.52604331865, + "max": 373406.21364514437 + }, + "recv_fps": { + "median": 20003.99958246546, + "min": 20002.39666995295, + "max": 22082.540562045557 + }, + "groups_lost": { + "median": 0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 2.73, + "min": 2.7, + "max": 11.77 + } + } + }, + "base-28a8c9a7a665": { + "4": { + "cells": [ + { + "cell": "relay/base-28a8c9a7a665/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 118042.99865044883, + "cycles_per_frame": 290218.0634557764, + "recv_fps": 19998.43619764393, + "groups_lost": 0, + "host_load_1m": 4.28 + }, + { + "cell": "relay/base-28a8c9a7a665/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 118914.85978383935, + "cycles_per_frame": 248664.0862251522, + "recv_fps": 20115.45945858679, + "groups_lost": 0, + "host_load_1m": 4.08 + }, + { + "cell": "relay/base-28a8c9a7a665/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 120613.85590159732, + "cycles_per_frame": 261572.58603313737, + "recv_fps": 20000.002939257127, + "groups_lost": 0, + "host_load_1m": 2.17 + } + ], + "instructions_per_frame": { + "median": 118914.85978383935, + "min": 118042.99865044883, + "max": 120613.85590159732 + }, + "cycles_per_frame": { + "median": 261572.58603313737, + "min": 248664.0862251522, + "max": 290218.0634557764 + }, + "recv_fps": { + "median": 20000.002939257127, + "min": 19998.43619764393, + "max": 20115.45945858679 + }, + "groups_lost": { + "median": 0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 4.08, + "min": 2.17, + "max": 4.28 + } + } + }, + "base-9f9149812ce3": { + "4": { + "cells": [ + { + "cell": "relay/base-9f9149812ce3/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 128498.55222376293, + "cycles_per_frame": 347032.1749611143, + "recv_fps": 19997.450219863327, + "groups_lost": 0, + "host_load_1m": 8.69 + }, + { + "cell": "relay/base-9f9149812ce3/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 130880.4116913851, + "cycles_per_frame": 343204.40250901255, + "recv_fps": 20001.15175504808, + "groups_lost": 0, + "host_load_1m": 5.27 + }, + { + "cell": "relay/base-9f9149812ce3/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 127980.50962678353, + "cycles_per_frame": 302061.7214663097, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 2.26 + } + ], + "instructions_per_frame": { + "median": 128498.55222376293, + "min": 127980.50962678353, + "max": 130880.4116913851 + }, + "cycles_per_frame": { + "median": 343204.40250901255, + "min": 302061.7214663097, + "max": 347032.1749611143 + }, + "recv_fps": { + "median": 20000.0, + "min": 19997.450219863327, + "max": 20001.15175504808 + }, + "groups_lost": { + "median": 0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 5.27, + "min": 2.26, + "max": 8.69 + } + } + }, + "base-a9a85580a448": { + "4": { + "cells": [ + { + "cell": "relay/base-a9a85580a448/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 129493.88201309738, + "cycles_per_frame": 334784.36443489586, + "recv_fps": 19997.46097568084, + "groups_lost": 0, + "host_load_1m": 3.25 + }, + { + "cell": "relay/base-a9a85580a448/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 129589.15040874497, + "cycles_per_frame": 338499.8624495248, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 7.38 + }, + { + "cell": "relay/base-a9a85580a448/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 128382.13453681872, + "cycles_per_frame": 292622.98673249676, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 3.5 + } + ], + "instructions_per_frame": { + "median": 129493.88201309738, + "min": 128382.13453681872, + "max": 129589.15040874497 + }, + "cycles_per_frame": { + "median": 334784.36443489586, + "min": 292622.98673249676, + "max": 338499.8624495248 + }, + "recv_fps": { + "median": 20000.0, + "min": 19997.46097568084, + "max": 20000.0 + }, + "groups_lost": { + "median": 0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 3.5, + "min": 3.25, + "max": 7.38 + } + } + }, + "base-43fc9251d71b": { + "4": { + "cells": [ + { + "cell": "relay/base-43fc9251d71b/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 126891.66269082314, + "cycles_per_frame": 347672.0702713368, + "recv_fps": 20014.962321356285, + "groups_lost": 0, + "host_load_1m": 12.02 + }, + { + "cell": "relay/base-43fc9251d71b/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 126354.89476991726, + "cycles_per_frame": 271712.8641892416, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 4.18 + }, + { + "cell": "relay/base-43fc9251d71b/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 133754.46697983646, + "cycles_per_frame": 296675.9455818364, + "recv_fps": 20069.323844040424, + "groups_lost": 0, + "host_load_1m": 2.96 + } + ], + "instructions_per_frame": { + "median": 126891.66269082314, + "min": 126354.89476991726, + "max": 133754.46697983646 + }, + "cycles_per_frame": { + "median": 296675.9455818364, + "min": 271712.8641892416, + "max": 347672.0702713368 + }, + "recv_fps": { + "median": 20014.962321356285, + "min": 20000.0, + "max": 20069.323844040424 + }, + "groups_lost": { + "median": 0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 4.18, + "min": 2.96, + "max": 12.02 + } + } + }, + "current": { + "4": { + "cells": [ + { + "cell": "relay/current/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 139008.70201293853, + "cycles_per_frame": 304073.4396178103, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 10.76 + }, + { + "cell": "relay/current/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 139155.4620059375, + "cycles_per_frame": 358026.6577482179, + "recv_fps": 19997.131949881437, + "groups_lost": 0, + "host_load_1m": 3.91 + }, + { + "cell": "relay/current/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 136824.79155837605, + "cycles_per_frame": 336991.56119588704, + "recv_fps": 19999.034719131432, + "groups_lost": 0, + "host_load_1m": 2.25 + } + ], + "instructions_per_frame": { + "median": 139008.70201293853, + "min": 136824.79155837605, + "max": 139155.4620059375 + }, + "cycles_per_frame": { + "median": 336991.56119588704, + "min": 304073.4396178103, + "max": 358026.6577482179 + }, + "recv_fps": { + "median": 19999.034719131432, + "min": 19997.131949881437, + "max": 20000.0 + }, + "groups_lost": { + "median": 0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 3.91, + "min": 2.25, + "max": 10.76 + } + } + } + } + }, + { + "artifact": "nixos2:run.S0G1Z7", + "host": "nixos2 (i7-8700; no quest gate there, so each run started after 80 s at load under 2.5 with no other process above half a core)", + "relay_cpus": "0-3", + "generator_cpus": "4,5,10,11", + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "workload": "chat", + "runtime": "io-uring-workers", + "revisions": { + "base": "cdff3c8f95, the old endpoint", + "current": "fa9c1bf2a7 (main)" + }, + "note": "main against the old endpoint", + "summary": { + "base": { + "4": { + "cells": [ + { + "cell": "relay/base/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 110438.19032281345, + "cycles_per_frame": 238151.8534170224, + "recv_fps": 19998.415557527132, + "groups_lost": 0, + "host_load_1m": 2.25 + }, + { + "cell": "relay/base/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 111769.07599916575, + "cycles_per_frame": 229119.09989778293, + "recv_fps": 20305.118326832566, + "groups_lost": 0, + "host_load_1m": 2.27 + }, + { + "cell": "relay/base/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 113222.34247587262, + "cycles_per_frame": 231797.83643476115, + "recv_fps": 19798.970409424925, + "groups_lost": 0, + "host_load_1m": 2.06 + }, + { + "cell": "relay/base/round-4/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 109419.11777739269, + "cycles_per_frame": 256960.96516555574, + "recv_fps": 20003.99965745121, + "groups_lost": 0, + "host_load_1m": 8.53 + } + ], + "instructions_per_frame": { + "median": 111103.6331609896, + "min": 109419.11777739269, + "max": 113222.34247587262 + }, + "cycles_per_frame": { + "median": 234974.84492589178, + "min": 229119.09989778293, + "max": 256960.96516555574 + }, + "recv_fps": { + "median": 20001.207607489174, + "min": 19798.970409424925, + "max": 20305.118326832566 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 2.26, + "min": 2.06, + "max": 8.53 + } + } + }, + "current": { + "4": { + "cells": [ + { + "cell": "relay/current/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 142874.01496667342, + "cycles_per_frame": 297412.72879827407, + "recv_fps": 20001.385834326353, + "groups_lost": 0, + "host_load_1m": 2.41 + }, + { + "cell": "relay/current/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 142595.95452064698, + "cycles_per_frame": 294528.8243329015, + "recv_fps": 20001.289157928073, + "groups_lost": 0, + "host_load_1m": 2.32 + }, + { + "cell": "relay/current/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 142700.63409929347, + "cycles_per_frame": 283296.1793441754, + "recv_fps": 19996.000511653296, + "groups_lost": 0, + "host_load_1m": 1.81 + }, + { + "cell": "relay/current/round-4/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 140914.0745016863, + "cycles_per_frame": 346765.42809844256, + "recv_fps": 20001.378613786146, + "groups_lost": 0, + "host_load_1m": 2.62 + } + ], + "instructions_per_frame": { + "median": 142648.29430997022, + "min": 140914.0745016863, + "max": 142874.01496667342 + }, + "cycles_per_frame": { + "median": 295970.7765655878, + "min": 283296.1793441754, + "max": 346765.42809844256 + }, + "recv_fps": { + "median": 20001.333885857108, + "min": 19996.000511653296, + "max": 20001.385834326353 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 2.365, + "min": 1.81, + "max": 2.62 + } + } + } + } + }, + { + "artifact": "nixos2:run.4U16wN", + "host": "nixos2 (i7-8700; no quest gate there, so each run started after 80 s at load under 2.5 with no other process above half a core)", + "relay_cpus": "0-3", + "generator_cpus": "4,5,10,11", + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "workload": "chat", + "runtime": "tokio-workers", + "revisions": { + "base": "cdff3c8f95, the old endpoint", + "current": "fa9c1bf2a7 (main)" + }, + "note": "main against the old endpoint", + "summary": { + "base": { + "4": { + "cells": [ + { + "cell": "relay/base/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 133810.68780368034, + "cycles_per_frame": 272289.31145911576, + "recv_fps": 20001.150776600938, + "groups_lost": 0, + "host_load_1m": 1.54 + }, + { + "cell": "relay/base/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 119722.746169692, + "cycles_per_frame": 395688.35060333286, + "recv_fps": 19227.27425200927, + "groups_lost": 0, + "host_load_1m": 8.69 + }, + { + "cell": "relay/base/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 131174.44382611386, + "cycles_per_frame": 402073.21234437416, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 5.91 + }, + { + "cell": "relay/base/round-4/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 134112.6098387605, + "cycles_per_frame": 281832.69365063467, + "recv_fps": 19998.787357937494, + "groups_lost": 0, + "host_load_1m": 2.14 + } + ], + "instructions_per_frame": { + "median": 132492.5658148971, + "min": 119722.746169692, + "max": 134112.6098387605 + }, + "cycles_per_frame": { + "median": 338760.52212698373, + "min": 272289.31145911576, + "max": 402073.21234437416 + }, + "recv_fps": { + "median": 19999.393678968747, + "min": 19227.27425200927, + "max": 20001.150776600938 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 4.025, + "min": 1.54, + "max": 8.69 + } + } + }, + "current": { + "4": { + "cells": [ + { + "cell": "relay/current/round-1/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 144457.64693984922, + "cycles_per_frame": 472152.72277027817, + "recv_fps": 20003.25572264157, + "groups_lost": 0, + "host_load_1m": 5.69 + }, + { + "cell": "relay/current/round-2/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 147217.42481670858, + "cycles_per_frame": 407840.37111283914, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 1.63 + }, + { + "cell": "relay/current/round-3/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 146580.8815739602, + "cycles_per_frame": 344608.2208468801, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 4.09 + }, + { + "cell": "relay/current/round-4/instructions-4/chat/instructions/instructions.json", + "instructions_per_frame": 147538.51930820433, + "cycles_per_frame": 415631.59350481915, + "recv_fps": 20000.0, + "groups_lost": 0, + "host_load_1m": 2.97 + } + ], + "instructions_per_frame": { + "median": 146899.1531953344, + "min": 144457.64693984922, + "max": 147538.51930820433 + }, + "cycles_per_frame": { + "median": 411735.98230882915, + "min": 344608.2208468801, + "max": 472152.72277027817 + }, + "recv_fps": { + "median": 20000.0, + "min": 20000.0, + "max": 20003.25572264157 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 3.5300000000000002, + "min": 1.63, + "max": 5.69 + } + } + } + } + }, + { + "artifact": "nixos2:run.kQW0En", + "host": "nixos2 (i7-8700; no quest gate there, so each run started after 80 s at load under 2.5 with no other process above half a core)", + "relay_cpus": "0-3", + "generator_cpus": "4,5,10,11", + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "workload": "video", + "runtime": "io-uring-workers", + "revisions": { + "base": "cdff3c8f95, the old endpoint", + "current": "fa9c1bf2a7 (main)" + }, + "note": "main against the old endpoint; superseded by run.TZ2ODR, since load rose to 12 from round 3", + "summary": { + "base": { + "4": { + "cells": [ + { + "cell": "relay/base/round-1/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 100959.7083146826, + "cycles_per_frame": 258488.1328036645, + "recv_fps": 959.9999999999998, + "groups_lost": 0, + "host_load_1m": 3.13 + }, + { + "cell": "relay/base/round-2/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 104398.56303404094, + "cycles_per_frame": 409492.47510648845, + "recv_fps": 960.0, + "groups_lost": 0, + "host_load_1m": 5.55 + }, + { + "cell": "relay/base/round-3/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 100008.31069186589, + "cycles_per_frame": 393307.08846508915, + "recv_fps": 960.0687760432135, + "groups_lost": 0, + "host_load_1m": 5.73 + }, + { + "cell": "relay/base/round-4/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 101293.57005249792, + "cycles_per_frame": 305725.30442575464, + "recv_fps": 960.0725854598239, + "groups_lost": 0, + "host_load_1m": 12.41 + } + ], + "instructions_per_frame": { + "median": 101126.63918359026, + "min": 100008.31069186589, + "max": 104398.56303404094 + }, + "cycles_per_frame": { + "median": 349516.1964454219, + "min": 258488.1328036645, + "max": 409492.47510648845 + }, + "recv_fps": { + "median": 960.0343880216067, + "min": 959.9999999999998, + "max": 960.0725854598239 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 5.640000000000001, + "min": 3.13, + "max": 12.41 + } + } + }, + "current": { + "4": { + "cells": [ + { + "cell": "relay/current/round-1/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 109050.8320884069, + "cycles_per_frame": 278462.7706222729, + "recv_fps": 959.9186614243695, + "groups_lost": 0, + "host_load_1m": 4.46 + }, + { + "cell": "relay/current/round-2/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 106514.81210849703, + "cycles_per_frame": 233454.70424598941, + "recv_fps": 959.9236715623178, + "groups_lost": 0, + "host_load_1m": 4.3 + }, + { + "cell": "relay/current/round-3/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 118047.2329899198, + "cycles_per_frame": 472909.05677116336, + "recv_fps": 959.9285481880923, + "groups_lost": 0, + "host_load_1m": 7.58 + }, + { + "cell": "relay/current/round-4/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 119511.52959884486, + "cycles_per_frame": 509520.2006930177, + "recv_fps": 959.8190288946225, + "groups_lost": 0, + "host_load_1m": 9.94 + } + ], + "instructions_per_frame": { + "median": 113549.03253916334, + "min": 106514.81210849703, + "max": 119511.52959884486 + }, + "cycles_per_frame": { + "median": 375685.91369671817, + "min": 233454.70424598941, + "max": 509520.2006930177 + }, + "recv_fps": { + "median": 959.9211664933437, + "min": 959.8190288946225, + "max": 959.9285481880923 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 6.02, + "min": 4.3, + "max": 9.94 + } + } + } + } + }, + { + "artifact": "nixos2:run.TZ2ODR", + "host": "nixos2 (i7-8700; no quest gate there, so each run started after 80 s at load under 2.5 with no other process above half a core)", + "relay_cpus": "0-3", + "generator_cpus": "4,5,10,11", + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "workload": "video", + "runtime": "io-uring-workers", + "revisions": { + "base": "cdff3c8f95, the old endpoint", + "current": "fa9c1bf2a7 (main)" + }, + "note": "main against the old endpoint", + "summary": { + "base": { + "4": { + "cells": [ + { + "cell": "relay/base/round-1/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 102236.731368789, + "cycles_per_frame": 237388.07399811584, + "recv_fps": 960.0000000000002, + "groups_lost": 0, + "host_load_1m": 1.08 + }, + { + "cell": "relay/base/round-2/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 98381.7177726354, + "cycles_per_frame": 219165.26216945075, + "recv_fps": 960.0000000000002, + "groups_lost": 0, + "host_load_1m": 0.69 + }, + { + "cell": "relay/base/round-3/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 97403.11236380997, + "cycles_per_frame": 229461.95448024358, + "recv_fps": 960.0005559515195, + "groups_lost": 0, + "host_load_1m": 0.56 + }, + { + "cell": "relay/base/round-4/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 99946.03189495618, + "cycles_per_frame": 220657.50927024856, + "recv_fps": 959.9999999999998, + "groups_lost": 0, + "host_load_1m": 0.51 + } + ], + "instructions_per_frame": { + "median": 99163.8748337958, + "min": 97403.11236380997, + "max": 102236.731368789 + }, + "cycles_per_frame": { + "median": 225059.73187524607, + "min": 219165.26216945075, + "max": 237388.07399811584 + }, + "recv_fps": { + "median": 960.0000000000002, + "min": 959.9999999999998, + "max": 960.0005559515195 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 0.625, + "min": 0.51, + "max": 1.08 + } + } + }, + "current": { + "4": { + "cells": [ + { + "cell": "relay/current/round-1/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 112029.15147321289, + "cycles_per_frame": 245347.81080353385, + "recv_fps": 959.967891094116, + "groups_lost": 0, + "host_load_1m": 0.86 + }, + { + "cell": "relay/current/round-2/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 112939.94528628506, + "cycles_per_frame": 256436.37128312435, + "recv_fps": 960.0, + "groups_lost": 0, + "host_load_1m": 0.76 + }, + { + "cell": "relay/current/round-3/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 108643.29631033321, + "cycles_per_frame": 236899.23952598218, + "recv_fps": 960.0000000000002, + "groups_lost": 0, + "host_load_1m": 0.7 + }, + { + "cell": "relay/current/round-4/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 109540.95851889307, + "cycles_per_frame": 229321.15657694146, + "recv_fps": 959.9999999999998, + "groups_lost": 0, + "host_load_1m": 0.58 + } + ], + "instructions_per_frame": { + "median": 110785.05499605298, + "min": 108643.29631033321, + "max": 112939.94528628506 + }, + "cycles_per_frame": { + "median": 241123.52516475803, + "min": 229321.15657694146, + "max": 256436.37128312435 + }, + "recv_fps": { + "median": 959.9999999999999, + "min": 959.967891094116, + "max": 960.0000000000002 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 0.73, + "min": 0.58, + "max": 0.86 + } + } + } + } + }, + { + "artifact": "nixos2:run.6xoc9u", + "host": "nixos2 (i7-8700; no quest gate there, so each run started after 80 s at load under 2.5 with no other process above half a core)", + "relay_cpus": "0-3", + "generator_cpus": "4,5,10,11", + "generator_revision": "21d6da9fc1", + "instruction_period": 2000003, + "workload": "video", + "runtime": "tokio-workers", + "revisions": { + "base": "cdff3c8f95, the old endpoint", + "current": "fa9c1bf2a7 (main)" + }, + "note": "main against the old endpoint", + "summary": { + "base": { + "4": { + "cells": [ + { + "cell": "relay/base/round-1/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 137396.5241378377, + "cycles_per_frame": 296358.83229715395, + "recv_fps": 959.9999999999998, + "groups_lost": 0, + "host_load_1m": 1.05 + }, + { + "cell": "relay/base/round-2/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 139800.1079917114, + "cycles_per_frame": 324778.4075747499, + "recv_fps": 960.0327677356548, + "groups_lost": 0, + "host_load_1m": 0.98 + }, + { + "cell": "relay/base/round-3/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 135610.50480158598, + "cycles_per_frame": 283721.6724235282, + "recv_fps": 959.8858989925936, + "groups_lost": 0, + "host_load_1m": 0.74 + }, + { + "cell": "relay/base/round-4/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 128660.72410306365, + "cycles_per_frame": 359705.8359279434, + "recv_fps": 960.0, + "groups_lost": 0, + "host_load_1m": 1.23 + } + ], + "instructions_per_frame": { + "median": 136503.51446971184, + "min": 128660.72410306365, + "max": 139800.1079917114 + }, + "cycles_per_frame": { + "median": 310568.6199359519, + "min": 283721.6724235282, + "max": 359705.8359279434 + }, + "recv_fps": { + "median": 959.9999999999999, + "min": 959.8858989925936, + "max": 960.0327677356548 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 1.0150000000000001, + "min": 0.74, + "max": 1.23 + } + } + }, + "current": { + "4": { + "cells": [ + { + "cell": "relay/current/round-1/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 121758.14733105317, + "cycles_per_frame": 283063.2078622138, + "recv_fps": 960.0, + "groups_lost": 0, + "host_load_1m": 0.85 + }, + { + "cell": "relay/current/round-2/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 118560.65847992722, + "cycles_per_frame": 269652.2327772885, + "recv_fps": 960.0000000000002, + "groups_lost": 0, + "host_load_1m": 1.23 + }, + { + "cell": "relay/current/round-3/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 121405.46336715626, + "cycles_per_frame": 281072.26724168874, + "recv_fps": 960.0, + "groups_lost": 0, + "host_load_1m": 1.53 + }, + { + "cell": "relay/current/round-4/instructions-4/video/instructions/instructions.json", + "instructions_per_frame": 124092.18159707768, + "cycles_per_frame": 298452.0986302454, + "recv_fps": 960.0, + "groups_lost": 0, + "host_load_1m": 1.59 + } + ], + "instructions_per_frame": { + "median": 121581.80534910472, + "min": 118560.65847992722, + "max": 124092.18159707768 + }, + "cycles_per_frame": { + "median": 282067.73755195126, + "min": 269652.2327772885, + "max": 298452.0986302454 + }, + "recv_fps": { + "median": 960.0, + "min": 960.0, + "max": 960.0000000000002 + }, + "groups_lost": { + "median": 0.0, + "min": 0, + "max": 0 + }, + "host_load_1m": { + "median": 1.38, + "min": 0.85, + "max": 1.59 + } + } + } + } } ], "notes": [ diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md index 3b5bfba0cf0..b7ea11130d6 100644 --- a/.github/relay-cpu-bisect.md +++ b/.github/relay-cpu-bisect.md @@ -1,12 +1,12 @@ # Relay CPU bisect -Relay instructions per delivered frame between [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) ([moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) as merged) and `main` at [c68c931025](https://github.com/moq-dev/moq/commit/c68c931025). Per-cell counts, delivery, loss, host load, and source markers are in [the evidence](relay-cpu-bisect.json). The [first checkpoint](https://github.com/moq-dev/moq/blob/8c62afd3195a3307054120226b54e3309a5dbc1f/.github/relay-cpu-bisect.md) and its [per-cell evidence](https://github.com/moq-dev/moq/blob/8c62afd3195a3307054120226b54e3309a5dbc1f/.github/relay-cpu-bisect.json) are superseded. +Relay instructions per delivered frame between [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) ([moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) as merged) and `main`: [c68c931025](https://github.com/moq-dev/moq/commit/c68c931025) for the bisect on the 24-core host, and [fa9c1bf2a7](https://github.com/moq-dev/moq/commit/fa9c1bf2a7) for the session re-measure and the re-pair on the quiet test host. Per-cell counts, delivery, loss, host load, and source markers are in [the evidence](relay-cpu-bisect.json). The [first checkpoint](https://github.com/moq-dev/moq/blob/8c62afd3195a3307054120226b54e3309a5dbc1f/.github/relay-cpu-bisect.md) and its [per-cell evidence](https://github.com/moq-dev/moq/blob/8c62afd3195a3307054120226b54e3309a5dbc1f/.github/relay-cpu-bisect.json) are superseded. ## Method `just bench-profile instructions` counts the relay's user and kernel instructions over a five-second window after seven seconds of warmup, divided by the frames the generator's 500 ms reports interpolate over the same window. Builds use the profiling profile with frame pointers and `moq-relay/io-uring,moq-bench/uring`; the relay runs four workers on cores 4-7 and the generator on cores 0-3. Chat is one publisher, 2,000 viewers, 10 fps, one 300-byte frame per group; video is the `video` workload at 960 delivered frames/s. -One run measures several revisions: every round counts each base and `main` back to back, rotating which goes first. Each revision builds once into its own target and checkout, so later runs reuse it. Every run passed through `quest gate`, and each cell records the host's highest one-minute load during its window. +One run measures several revisions: every round counts each base and `main` back to back, rotating which goes first. Each revision builds once into its own target and checkout, so later runs reuse it. Every run on the 24-core host passed through `quest gate`, and each cell records the host's highest one-minute load during its window. The runs on the quiet test host `nixos2`, which has no `quest`, each started only after 80 seconds at a one-minute load under 2.5 with no other process above half a core. Three findings shaped the method during this attempt: @@ -35,21 +35,23 @@ Markers are inclusive shares of the fixed-period samples; they overlap and are n - **[moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224), 26k of a +34k step, accidental, fixed.** Its `Reader::learn_end` polled each subscriber's finish through the track lock on every forward-drop pass (`poll_forward_drop` → `Subscriber::poll_drop` → `Reader::poll_serving` → `sync` → `learn_end`). It takes 10-18% of all relay instructions from [fd59eb18a5](https://github.com/moq-dev/moq/commit/fd59eb18a50f6e0ac819e73d941a3832cbdac91f), the PR's merge, through [2fff7ea7ae](https://github.com/moq-dev/moq/commit/2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3), and none at its parent ([run.16AJUb](relay-cpu-bisect.json) has every point in the interval). [moq-dev/moq#7532](https://github.com/moq-dev/moq/pull/7532) brought it to 0.5% on `main`. - **[moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171) and the changes after it, +39k to [2fff7ea7ae](https://github.com/moq-dev/moq/commit/2fff7ea7ae4eff1ac984c1f1af6894e3529f1cf3), mostly fixed.** Dynamic tracks added `Cursor::probe`, 15.8k at [991572d636](https://github.com/moq-dev/moq/commit/991572d636d264c6189a3477849d2969e689f55c) and 9.8k on `main` (below). The rest is track lock contention: futex syscalls and the forward-drop path. [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493) took the fanout subscribers off the shared lock, and `main`'s syscall leaf is now below the old endpoint's (26.3k against 28.8k). -- **[cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) to [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e), +18k, a staircase of feature steps.** The first checkpoint measured [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929)'s per-reader ranges and time anchors at +6k. Runs eVjz2w and 2DucdO (main's generator, load 2-5 in the rounds used) put about +6k across [moq-dev/moq#5928](https://github.com/moq-dev/moq/pull/5928) (compressed tracks) and [moq-dev/moq#6129](https://github.com/moq-dev/moq/pull/6129) (a subscriber reads only its ranges' frames), and +12k in steps of 2-4k from [2d73eea67d](https://github.com/moq-dev/moq/commit/2d73eea67d083ecf18938f4fc7c4842118eb87cf): qmux in tree ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)), signed priorities ([moq-dev/moq#6142](https://github.com/moq-dev/moq/pull/6142)), the m0 simplification audit and public ingest ([moq-dev/moq#6108](https://github.com/moq-dev/moq/pull/6108), [moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111)), and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)). Each step is within twice the spread between rounds, so these name the interval, not a single commit's cost. +- **[cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) to [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e), +18k, a staircase of feature steps.** The first checkpoint measured [moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929)'s per-reader ranges and time anchors at +6k. Runs eVjz2w and 2DucdO (main's generator, load 2-5 in the rounds used) put about +6k across [moq-dev/moq#5928](https://github.com/moq-dev/moq/pull/5928) (compressed tracks) and [moq-dev/moq#6129](https://github.com/moq-dev/moq/pull/6129) (a subscriber reads only its ranges' frames), and +12k across [2d73eea67d](https://github.com/moq-dev/moq/commit/2d73eea67d083ecf18938f4fc7c4842118eb87cf) to [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e). Relay runs on the quiet test host put that step on public ingest ([moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111), +10.5k), and none on in-tree qmux, receive leases, or signed priorities; see [below](#qmux-receive-leases-and-public-ingest-on-the-relay). - **[moq-dev/moq#7530](https://github.com/moq-dev/moq/pull/7530) moved io\_uring submission, not cost.** `udp::send` doubled after [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146) because each turn now submits its queued sends inside `Shared::push_inner`, while total syscall instructions fell from 39.2k to 26.3k per frame. ## Main against the old endpoint -Every runtime and workload, with the pinned generator: +Re-paired on the quiet test host `nixos2` (an i7-8700: relay on CPUs 0-3, generator on 4, 5, 10, and 11), against `main` at [fa9c1bf2a7](https://github.com/moq-dev/moq/commit/fa9c1bf2a7) with the probe fast path, the pinned generator, and a fixed sampling period of 2,000,003: -| Workload, runtime | Run | [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) | [21d6da9fc1](https://github.com/moq-dev/moq/commit/21d6da9fc108b48373a961f34e43c3fcd38b0146) | `main` | `main` / old | Highest load | -| --- | --- | ---: | ---: | ---: | ---: | ---: | -| chat, io\_uring | qYPSii, quiet round | 117.7 | 176.2 | 148.8 | 1.26 | 3 | -| chat, tokio workers | IV47ua, 4 rounds | 131.1 \[123.6-131.5] | 186.1 \[184.3-189.6] | 171.6 \[167.8-172.8] | 1.31 | 23 | -| video, io\_uring | SkAD0k, 4 rounds | 109.7 \[105.6-119.9] | 140.3 \[138.3-146.8] | 124.9 \[120.5-138.3] | 1.14 | 40 | -| video, tokio workers | Z04ENl, 4 rounds | 152.8 \[148.2-163.9] | 167.3 \[163.2-175.3] | 146.8 \[142.9-147.3] | 0.96 | 40 | +| Workload, runtime | Run | [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) | `main` | `main` / old | Highest load | +| --- | --- | ---: | ---: | ---: | ---: | +| chat, io\_uring | S0G1Z7 | 111.8 \[110.4-113.2] | 142.6 \[140.9-142.9] | 1.28 | 2.6 | +| chat, tokio workers | 4U16wN | 133.8 \[131.2-134.1] | 146.9 \[144.5-147.5] | 1.10 | 5.9 | +| video, io\_uring | TZ2ODR | 99.2 \[97.4-102.2] | 110.8 \[108.6-112.9] | 1.12 | 1.1 | +| video, tokio workers | 6xoc9u | 136.5 \[128.7-139.8] | 121.6 \[118.6-124.1] | 0.89 | 1.6 | -Medians of four rounds, min-max in brackets; every cell delivered its workload's full rate with no loss. The video rows ran at load up to 40, so treat their ratios as approximate. Chat rose on both runtimes, so the remaining cost is shared code. By the nearest named function, chat's growth on `main` falls in the same places on io\_uring (run.qYPSii) and the tokio workers (run.IV47ua), in instructions per frame: +Medians of the quiet cells in four rounds, min-max in brackets: every cell used delivered its workload's rate within 2% with no loss, at a one-minute load of at most 6. Two cells were dropped: the old endpoint's fourth chat round on io\_uring (load 8.5) and its second on the tokio workers (load 8.7, 3.9% short of the rate). Run 4U16wN's `main` and run ssZrgC's (142.3 \[141.6-143.1], three rounds) agree within 0.3k with run S0G1Z7's. + +The earlier pairing on the 24-core host, at load up to 40 for the video rows, read chat 1.26 on io\_uring and 1.31 on the tokio workers, and video 1.14 and 0.96. Only chat on the tokio workers moved by more than those runs' spread, and its earlier run reached load 23 on another host, so the change is not attributed. Chat on io\_uring remains the workload furthest above the old endpoint. By the nearest named function, chat's growth on `main` at [c68c931025](https://github.com/moq-dev/moq/commit/c68c931025), before the fast path, falls in the same places on io\_uring (run.qYPSii) and the tokio workers (run.IV47ua), in instructions per frame: | Growth on `main` | io\_uring | tokio | Introduced by | | --- | ---: | ---: | --- | @@ -79,7 +81,7 @@ At 512 subscribers, in instructions per delivered group, the model path cost 4,2 | Public ingest, [moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111) | 5,816 | 6,016 | +200 | | The m0 simplification audit, [moq-dev/moq#6108](https://github.com/moq-dev/moq/pull/6108) | 6,017 | 6,017 | 0 | -In-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)) are QUIC-level, which neither scenario reaches. A relay run pairing each with its parent on the quiet test host got one round (qmux 116k to 122k, leases 130k to 132k per frame) before other agents started fuzzing and compiling there, so neither is attributed yet. The model path barely moved before [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e) (5,840) and then grew in a few large steps, each against its parent (run.TmGdFy, one build profile throughout): +In-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)) are QUIC-level, which neither scenario reaches. Relay runs attribute them [below](#qmux-receive-leases-and-public-ingest-on-the-relay). The model path barely moved before [e84ffb3de3](https://github.com/moq-dev/moq/commit/e84ffb3de3139f7245bed5697754fd6add9a022e) (5,840) and then grew in a few large steps, each against its parent (run.TmGdFy, one build profile throughout): | Commit | Model | Session | | --- | ---: | ---: | @@ -134,6 +136,32 @@ Neither cost is in the probe's judgement, so the fast path left both in place: | Model | 9,637 | 9,367 | -270 (-2.8%) | | Session | 110,014 | 109,123 | -891 (-0.8%) | +## qmux, receive leases, and public ingest on the relay + +In-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)) are QUIC-level, so only relay runs reach them. On the quiet test host, chat over io\_uring, each against its parent in one run (run.ssZrgC, three rounds, load at most 3.5): + +| Change | Parent | Merge | Change | +| --- | ---: | ---: | ---: | +| In-tree qmux, [moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231) | 121.3 \[120.2-123.0] | 119.4 \[116.5-122.6] | -1.9 | +| Receive leases, [moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878) | 134.4 \[130.9-134.4] | 135.2 \[135.1-136.5] | +0.8 | + +Neither costs measurably: each change is inside its own rounds' spread. The rise between them, +15.0k in that run, sits at public ingest. Run YxLYgw measured every candidate in that interval in one run (three rounds; the medians use the cells at load 6 or less, and the step shows in every round, loaded cells included): + +| Revision | kinstr/frame | +| --- | ---: | +| [72259ab3cc](https://github.com/moq-dev/moq/commit/72259ab3cc), in-tree qmux merged | 119.5 | +| [d44813cb70](https://github.com/moq-dev/moq/commit/d44813cb70), before signed priorities | 120.5 \[118.5-122.4] | +| [b4dfafac88](https://github.com/moq-dev/moq/commit/b4dfafac88), signed priorities, [moq-dev/moq#6142](https://github.com/moq-dev/moq/pull/6142) | 119.6 \[117.7-121.5] | +| [1f5caa7026](https://github.com/moq-dev/moq/commit/1f5caa7026), a GSO batch ends on a short first datagram, [moq-dev/moq#6279](https://github.com/moq-dev/moq/pull/6279) | 120.5 \[118.7-122.3] | +| [28a8c9a7a6](https://github.com/moq-dev/moq/commit/28a8c9a7a6), before public ingest | 118.9 \[118.0-120.6] | +| [9f9149812c](https://github.com/moq-dev/moq/commit/9f9149812c), public ingest, [moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111) | 129.4 \[128.0-130.9] | +| [a9a85580a4](https://github.com/moq-dev/moq/commit/a9a85580a4), the m0 simplification audit, [moq-dev/moq#6108](https://github.com/moq-dev/moq/pull/6108) | 128.9 \[128.4-129.5] | +| [43fc9251d7](https://github.com/moq-dev/moq/commit/43fc9251d7), before receive leases | 130.1 \[126.4-133.8] | + +Public ingest added +10.5k per frame here, against +200 per group through the model, because the model scenario never reads a frame. It stopped withholding groups: every group joins arrival order when created, so each viewer's read reaches a group before its first frame and blocks on it. By function (rounds 2 and 3), the rise is each viewer's group serving (`GroupServe`, +5.3k inclusive: `group::Consumer::poll_read_frame` and the new `poll_next_frame`, with the expiry checks a blocked read runs, `poll_expired_if_blocked`, `poll_expired_while_pending`, `Recover::poll_budget`, and `watch_landing`), new track-state lock polls in the forward-drop, receive, and staleness paths (+3.8k), and the lock and unlock leaves (+2.0k self). + +On `main`, the in-process session's group read (`group::Consumer::poll_read_objects`) costs 4.5k per delivered group (run 4rAIYS), of which `GroupExpiry::is_expired` is 1.6k, `Consumer::poll_expiry` 1.4k, and `Recover::poll_budget` 1.0k. `poll_expiry` asks the budget before `is_expired`, which answers false without it while nothing landed above the group, but the budget's poll also syncs the reader's mirrored preferences, so skipping it is not a change that can be made without proving that sync redundant. + ## Validation -Every count, delivery estimate, loss figure, and host load cited here is in [the evidence](relay-cpu-bisect.json), keyed by run artifact. Raw captures (`perf.data`, folded stacks, reports) stay in the measuring checkout's ignored `.tmp/bench/`. +Every count, delivery estimate, loss figure, and host load cited here is in [the evidence](relay-cpu-bisect.json), keyed by run artifact. Raw captures (`perf.data`, folded stacks, reports) stay in the measuring checkout's ignored `.tmp/bench/`; the per-function figures for public ingest are recorded with run YxLYgw. diff --git a/quest/m1/release-perf/README.md b/quest/m1/release-perf/README.md index 3fe376bf309..2cc592c8784 100644 --- a/quest/m1/release-perf/README.md +++ b/quest/m1/release-perf/README.md @@ -330,6 +330,13 @@ per delivered group at 512 subscribers through the in-process session: cost 2,117 on `main`, about 40 queries per delivery at about 50 instructions each: per-reader windows answered by search, not a walk. +In-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) +and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)) +add no measurable relay cost on chat over io\_uring, each against its parent +on the quiet test host, so neither needs accepting. Public ingest's relay cost +is larger than the model counted and waits on a decision in the +[bisect](/quest/m1/release-perf/relay-cpu-bisect.md). + ## Required - [Relay CPU per frame since #5926](/quest/m1/release-perf/relay-cpu-bisect.md) - the 18-56% rise in relay instructions per delivered frame is bisected, attributed, and its avoidable cost fixed diff --git a/quest/m1/release-perf/relay-cpu-bisect.md b/quest/m1/release-perf/relay-cpu-bisect.md index e4f6313f667..000f843dadc 100644 --- a/quest/m1/release-perf/relay-cpu-bisect.md +++ b/quest/m1/release-perf/relay-cpu-bisect.md @@ -8,53 +8,25 @@ The commits that raised the relay's instructions per delivered frame between [cd Found by [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505). Decided 2026-11-01 (maintainer): m1, in this epic, since `main` must not reach the cut carrying accidental per-frame cost. -Bisected in [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605), which committed the driver (`just bench-profile instructions`, several rotated bases per run, a pinned `BENCH_GENERATOR`, per-cell host load) and [the report](/.github/relay-cpu-bisect.md). On chat over io_uring, the relay peaked at 1.82x the old endpoint and `main` is at 1.26x: +[The report](/.github/relay-cpu-bisect.md) has every step, its cause, and its evidence. [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605) bisected the rise, fixed the accidental `Cursor::probe` work, and recorded the costs the maintainer accepted on 2026-11-03 in the epic. [moq-dev/moq#7874](https://github.com/moq-dev/moq/pull/7874) finished the remaining measurements: -- [moq-dev/moq#6224](https://github.com/moq-dev/moq/pull/6224)'s `Reader::learn_end` polling (+26k instructions per frame) and the track lock contention after [moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171) were accidental. [moq-dev/moq#7532](https://github.com/moq-dev/moq/pull/7532) and [moq-dev/moq#7493](https://github.com/moq-dev/moq/pull/7493) already removed both from `main`. -- `main` against the old endpoint: chat 1.26x on io_uring and 1.31x on the tokio workers, video 1.14x on io_uring and 0.96x on the tokio workers. Chat's growth is shared code, in the same functions on both runtimes. +- [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995)'s and [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542)'s session costs were never the probe's judgement, so the fast path left both. One branch per live walk took back 891 per group; the rest is accepted in the epic. +- In-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)) cost nothing measurable on the relay. The step between them is public ingest ([moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111)): +10.5k per frame on chat over io\_uring, in each viewer's frame read. +- `main` against the old endpoint, on the quiet test host: chat 1.28x on io\_uring and 1.10x on the tokio workers, video 1.12x and 0.89x. -Decided 2026-11-03 (maintainer, on [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)): +What remains is the decision below on public ingest's relay cost. The epic accepted public ingest on 2026-11-03 at its model cost (+200 per group), which never reads a frame. -1. `Cursor::probe` on every peek ([moq-dev/moq#6171](https://github.com/moq-dev/moq/pull/6171), about 8.6k instructions per frame): - - ✅ Add a fast path in this quest that skips the work while nothing is missable. - - Accept it as the cost of relay backfill. -2. The cursor's window, drift, and live-edge reads ([moq-dev/moq#5929](https://github.com/moq-dev/moq/pull/5929), about 9-11k) and the 2-4k steps from compressed tracks, ranged reads, in-tree qmux, signed priorities, public ingest, and receive leases: - - Accept them as feature costs. - - ✅ Investigate each: profile each step's added instructions, fix what can be cut without changing behavior, and attribute what can't with the evidence. -3. kio's task and waiter growth (about 11k) and the lite publisher's serving and close polling (about 6k): - - ✅ Profile them in this quest, then fix or attribute them before closing. - - Hand them to the [performance audit](/quest/m2/audit-perf.md). +Measure as the report does. For a model or session change, `just bench-fanout` against the base, which host load doesn't move; the session scenario reads every frame, so it reaches public ingest's read path. For the relay, `BENCH_GENERATOR=21d6da9fc1`, `BENCH_INSTRUCTION_PERIOD=2000003`, `kernel.perf_event_max_sample_rate` raised to 100,000 and restored after, on a quiet host such as `nixos2` (relay on CPUs 0-3, generator on 4,5,10,11, `BENCH_BUILD_JOBS=2`), comparing only cells from one run. -`Cursor::probe` is the relay's check for groups a reader has missed. Each time a subscriber asks for its next group, the probe looks at what arrived on the track since its last look and decides whether any group the reader still wants has left the cache without the live feed delivering it (evicted, skipped, or from before the feed started), and asks upstream to fill those. A live reader that takes each group as it arrives never has such a gap, yet it paid for the whole check on every group: locking the subscription and the track's shared state, cloning the track's fills, walking the arrival ledger, checking each new group for holes, and running the ask-and-answer loop. On top of that, each time the cache aged out an old group, the eviction landed with every reader, all of which had already taken it, and forced each one to re-run the full check. - -Follow-up planning on 2026-11-03 (maintainer) added a load-independent mode to the driver: `just bench-fanout` counts the chat fan-out per delivered group on one pinned thread, through the model alone and through a relay over the in-memory transport, for any list of revisions. [The report](/.github/relay-cpu-bisect.md) attributes each step with it, and the fixes so far take `main`'s model path from 13,300 to 10,126 instructions per group (-23.9%) and the in-process session from 120,948 to 113,133 (-6.5%), at 512 subscribers. - -Decided 2026-11-03 (maintainer, on [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605)), after the attribution: - -1. Costs measured and not cut: - - ✅ Accept the flat ones (per-reader ranges' remainder, ranged reads, public ingest, compressed tracks, [moq-dev/moq#7052](https://github.com/moq-dev/moq/pull/7052), and bounded reads), recorded with their numbers in the [epic](/quest/m1/release-perf/README.md). Keep [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995)'s and [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542)'s session cost here for a re-measure against this branch. - - Accept all of them now. - - Keep investigating all of them. -2. The waiter's list records: - - ✅ A follow-up at m0, as a correctness bug: [a waiter's record limit changes no behavior](/quest/m0/kio-waiter-records.md). - - Leave the limit at 8. -3. `group::Consumer` moves: - - ✅ A follow-up in this epic, implemented in [moq-dev/moq#7791](https://github.com/moq-dev/moq/pull/7791). - - Leave it. - -Remaining: - -- Re-measure [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995)'s and [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542)'s session cost against `main` with the probe fast path (about +3.3k and +3.1k per group when each landed; much of it was the probe's walk of the live groups, which the fast path now skips), then fix what is accidental or record what remains in the epic. -- Attribute in-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)), each against its parent, with relay runs on a quiet host: they are QUIC-level, which `just bench-fanout` doesn't reach, and the one attempt on `nixos2` lost the host to other agents after a round. -- Re-pair `main` against [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) on chat and video over both runtimes on a quiet host, then delete this quest. +Public API: none expected. Wire: none. -Measure as the report does. For a model or session change, `just bench-fanout` against the base, which host load doesn't move. For the relay, `BENCH_GENERATOR=21d6da9fc1`, `BENCH_INSTRUCTION_PERIOD=2000003`, `kernel.perf_event_max_sample_rate` raised to 100,000 and restored after, on a quiet host such as `nixos2` (6 cores: relay on 0-3, generator on 4,5,10,11, `BENCH_BUILD_JOBS=2` for its 15 GiB), comparing only cells from one run; instructions per frame fall by up to 20% when other host work makes the relay batch more frames per wakeup. +## Waiting -Public API: none expected. Wire: none. +- Maintainer: decide public ingest's relay cost ([moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111), +10.5k of `main`'s +30.8k per frame on chat over io\_uring). Each viewer's read now reaches a group before its first frame and blocks on it, running the expiry checks a blocked read runs, and the forward-drop, receive, and staleness paths poll the track state more. Options: (1, recommended) keep this quest open to cut it, counting `main`'s per-viewer read with `just bench-fanout` session and callgrind, fixing what a live read pays for nothing, and recording the rest in the epic; (2) accept it as the price of readers reaching a group before its first frame, record it in the epic, and delete this quest; (3) hand it to the [performance audit](/quest/m2/audit-perf.md). Whether a feature's cost is worth more work is the maintainer's call, so no agent can take it. Done when this item is replaced by the pick. ## Related - [A waiter's record limit changes no behavior](/quest/m0/kio-waiter-records.md) - found here; the session's waiter registration cost waits on it - +- [Benchmark sessions can't overlap on a host](/quest/m0/bench-host-lock.md) - the quiet-host runs here were voided twice by other sessions' work - [Calibrate the paired nightly benchmarks](/quest/m1/bench-ci.md) - the nightly series that should have caught this growth - [Performance audit](/quest/m2/audit-perf.md) - counts instructions when bisecting, for the same reason From c6753bd267f892c62ec1a2b8bff25ab559a68d8a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 21:57:57 -0800 Subject: [PATCH 07/16] perf(net)!: a live read skips the reader's budget and an empty drop report A blocked group or frame read asked its logical reader for the budget before the expiry policy checked whether any group had landed above it. That check answers no without the budget, so the reader lock and preference sync were wasted on every blocked read at the live edge. The budget is now asked only once a verdict needs it; preference updates already mirror into every copy. A reader's drop poll judged its copies even with nothing to report. It now returns at once, since the serving copy's own poll wakes it for the next run. Together they cut 1,731 instructions per delivered group from the session fan-out at 512 viewers (callgrind, 105,676 to 103,946). group::Consumer::poll_next_frame is crate-private: nothing outside moq-net read through it. Adds a `group` fan-out scenario that counts group_relay_frames' drain. Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 6 +++- bench/fanout.py | 5 ++-- bench/fanout/group/Cargo.toml | 18 ++++++++++++ bench/fanout/group/src/main.rs | 34 +++++++++++++++++++++++ bench/run.sh | 4 +-- quest/m1/release-perf/relay-cpu-bisect.md | 8 ++---- rs/moq-net/MIGRATE.md | 4 +++ rs/moq-net/benches/group.rs | 10 +++---- rs/moq-net/src/model/frame.rs | 3 +- rs/moq-net/src/model/group.rs | 18 ++++++------ rs/moq-net/src/model/resume.rs | 16 ++++++++--- rs/moq-net/src/model/track.rs | 13 +++++---- 12 files changed, 105 insertions(+), 34 deletions(-) create mode 100644 bench/fanout/group/Cargo.toml create mode 100644 bench/fanout/group/src/main.rs diff --git a/bench/README.md b/bench/README.md index 27f6303029b..1cf227d824e 100644 --- a/bench/README.md +++ b/bench/README.md @@ -412,7 +412,8 @@ nix develop --command just bench-fanout "cdff3c8f95 21d6da9fc1" nix develop --command just bench-fanout "" "model" "1 512" ``` -Two scenarios run every subscriber count on one thread: +Each scenario runs every subscriber count on one thread. `model` and `session` +run by default: - `model`: one finished group a cycle through the track model, which every subscriber takes before parking again, as moq-net's `track_fanout_group` @@ -423,6 +424,9 @@ Two scenarios run every subscriber count on one thread: runtime. It counts the publisher, relay, and viewer sessions together, but no QUIC: in-tree qmux, receive leases, and crypto only show in relay counts. A revision whose session API predates this harness reports it unsupported. +- `group`, on request: every subscriber drains one finished 512-frame group in + batches, as moq-net's `group_relay_frames/plain/512/*` bench does, so that + bench's cost is counted without its wall-clock noise. Each `bench/fanout/` crate builds against each base's and the current tree's own crates and lock, `BENCH_BUILD_JOBS` at a time, into diff --git a/bench/fanout.py b/bench/fanout.py index 5e7c04840f2..af4ab8e284c 100644 --- a/bench/fanout.py +++ b/bench/fanout.py @@ -16,8 +16,9 @@ from pathlib import Path # Group deliveries per measurement: subscribers times cycles, so every subscriber count -# runs about as long. The session harness costs about ten times the model's per group. -UNITS = {"model": 1_000_000, "session": 100_000} +# runs about as long. The session harness costs about ten times the model's per group, and +# the group harness's 512-frame groups about five hundred times. +UNITS = {"model": 1_000_000, "session": 100_000, "group": 2_000} def cycles(scenario, subscribers): diff --git a/bench/fanout/group/Cargo.toml b/bench/fanout/group/Cargo.toml new file mode 100644 index 00000000000..02711998492 --- /dev/null +++ b/bench/fanout/group/Cargo.toml @@ -0,0 +1,18 @@ +# Built by `bench/run.sh --fanout` against each revision's moq-net, outside the workspace, +# so a historical checkout measures the same scenario. Its own lock is that revision's. +[package] +name = "fanout-group" +version = "0.0.0" +edition = "2024" +publish = false + +[workspace] + +[dependencies] +futures = "0.3" +moq-net = { path = "../../../rs/moq-net", features = ["fuzz"] } + +[profile.release] +codegen-units = 1 +debug = "line-tables-only" +lto = "fat" diff --git a/bench/fanout/group/src/main.rs b/bench/fanout/group/src/main.rs new file mode 100644 index 00000000000..b406869b9f5 --- /dev/null +++ b/bench/fanout/group/src/main.rs @@ -0,0 +1,34 @@ +//! A relay's frame fan-out: every subscriber drains one finished 512-frame group in batches, +//! as moq-net's `group_relay_frames/plain/512/*` bench does. +//! Usage: `fanout-group SUBSCRIBERS CYCLES`. The work is deterministic, so its retired +//! instructions, less a run of zero cycles, are the per-group cost whatever the host does. +use std::hint::black_box; + +use futures::FutureExt; +use moq_net::frame; + +const FRAMES: usize = 512; + +fn main() { + let mut args = std::env::args().skip(1).map(|a| a.parse::().unwrap()); + let (subscribers, cycles) = (args.next().unwrap(), args.next().unwrap()); + let (broadcast, track, group) = moq_net::fuzz::object_group(FRAMES, false); + let mut buffer = frame::Buffer::<16>::new(); + for _ in 0..cycles { + for _ in 0..subscribers { + let mut reader = group.consume(); + loop { + let batch = reader.read_frames(&mut buffer).now_or_never().unwrap().unwrap(); + if batch.is_empty() { + break; + } + for frame in batch.iter() { + black_box(frame); + } + } + } + } + // Left standing, so teardown stays out of the count. + std::mem::forget((broadcast, track, group)); + std::process::exit(0); +} diff --git a/bench/run.sh b/bench/run.sh index 655ea1e719d..12e3c7e32a4 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -142,8 +142,8 @@ case $MODE in FANOUT_SCENARIOS=${BENCH_FANOUT_SCENARIOS:-model session} FANOUT_SUBSCRIBERS=${BENCH_FANOUT_SUBSCRIBERS:-1 64 512} for value in $FANOUT_SCENARIOS; do - if [[ ! $value =~ ^(model|session)$ ]]; then - printf 'unknown fan-out scenario %q; expected model or session\n' "$value" >&2 + if [[ ! $value =~ ^(model|session|group)$ ]]; then + printf 'unknown fan-out scenario %q; expected model, session, or group\n' "$value" >&2 exit 1 fi done diff --git a/quest/m1/release-perf/relay-cpu-bisect.md b/quest/m1/release-perf/relay-cpu-bisect.md index 43b7f1eadac..38ffe428a09 100644 --- a/quest/m1/release-perf/relay-cpu-bisect.md +++ b/quest/m1/release-perf/relay-cpu-bisect.md @@ -14,7 +14,9 @@ Found by [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505). Decided 2 - In-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)) cost nothing measurable on the relay. The step between them is public ingest ([moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111)): +10.5k per frame on chat over io\_uring, in each viewer's frame read. - `main` against the old endpoint, on the quiet test host: chat 1.28x on io\_uring and 1.10x on the tokio workers, video 1.12x and 0.89x. -What remains is the decision below on public ingest's relay cost, and the `group_relay_frames` item below. The epic accepted public ingest on 2026-11-03 at its model cost (+200 per group), which never reads a frame. +What remains is public ingest's relay cost and the `group_relay_frames` item below. The epic accepted public ingest on 2026-11-03 at its model cost (+200 per group), which never reads a frame. + +Decided 2026-11-05 (maintainer, /quest-unblock): keep cutting public ingest's relay cost ([moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111), +10.5k of `main`'s +30.8k per frame on chat over io\_uring). Count `main`'s per-viewer read with `just bench-fanout` session and callgrind, fix what a live read pays for nothing, and record the rest in the epic. The listed lead is `Consumer::poll_expiry` calling `Recover::poll_budget` before `is_expired`; skipping it needs the mirrored-preference sync proven redundant. Not chosen: accepting the cost, or handing it to the [performance audit](/quest/m2/audit-perf.md). Also remove, or make crate-private, any public API on the relay read paths this work touches that no consumer outside its own crate uses. Measure as the report does. For a model or session change, `just bench-fanout` against the base, which host load doesn't move; the session scenario reads every frame, so it reaches public ingest's read path. For the relay, `BENCH_GENERATOR=21d6da9fc1`, `BENCH_INSTRUCTION_PERIOD=2000003`, `kernel.perf_event_max_sample_rate` raised to 100,000 and restored after, on a quiet host such as `nixos2` (relay on CPUs 0-3, generator on 4,5,10,11, `BENCH_BUILD_JOBS=2`), comparing only cells from one run. @@ -22,10 +24,6 @@ Added 2026-11-04 (maintainer) from [fastest-reader pacing](https://github.com/mo Public API: none expected. Wire: none. -## Waiting - -- Maintainer: decide public ingest's relay cost ([moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111), +10.5k of `main`'s +30.8k per frame on chat over io\_uring). Each viewer's read now reaches a group before its first frame and blocks on it, running the expiry checks a blocked read runs, and the forward-drop, receive, and staleness paths poll the track state more. Options: (1, recommended) keep this quest open to cut it, counting `main`'s per-viewer read with `just bench-fanout` session and callgrind, fixing what a live read pays for nothing, and recording the rest in the epic; (2) accept it as the price of readers reaching a group before its first frame, record it in the epic, and delete this quest; (3) hand it to the [performance audit](/quest/m2/audit-perf.md). Whether a feature's cost is worth more work is the maintainer's call, so no agent can take it. Done when this item is replaced by the pick. - ## Related - [A waiter's record limit changes no behavior](/quest/m0/kio-waiter-records.md) - found here; the session's waiter registration cost waits on it diff --git a/rs/moq-net/MIGRATE.md b/rs/moq-net/MIGRATE.md index 79fb80906f3..bfc49be6247 100644 --- a/rs/moq-net/MIGRATE.md +++ b/rs/moq-net/MIGRATE.md @@ -4,6 +4,10 @@ How to update your code for each breaking change, newest first. ## Unreleased +### `group::Consumer::poll_next_frame` is crate-private ([#7874](https://github.com/moq-dev/moq/pull/7874)) + +Read a frame in chunks with the async `group::Consumer::next_frame`, or read whole frames with `poll_read_frame` from a poll loop. + ### Opens wait their turn for stream credit ([#7894](https://github.com/moq-dev/moq/pull/7894)) A request no longer fails with `Error::NoStreamCredit` when the peer grants no more streams: every stream a session opens waits for credit, and returning credit goes to the open whose stream the transport would send first. The variant is removed; drop any arm matching it. diff --git a/rs/moq-net/benches/group.rs b/rs/moq-net/benches/group.rs index 368a682e39a..ecaf48ab9d0 100644 --- a/rs/moq-net/benches/group.rs +++ b/rs/moq-net/benches/group.rs @@ -779,14 +779,14 @@ fn streaming(b: &mut criterion::Bencher, chunk: &Bytes, arrivals: usize, threads loop { let reading = match frame { Some(frame) => frame, - None => match group.poll_next_frame(&waiter) { - Poll::Ready(Ok(Some(next))) => frame.insert(next), - Poll::Ready(Ok(None)) => { + None => match group.next_frame().now_or_never() { + Some(Ok(Some(next))) => frame.insert(next), + Some(Ok(None)) => { *group = next.lock().unwrap().clone(); continue; } - Poll::Ready(Err(err)) => panic!("next frame failed: {err}"), - Poll::Pending => break, + Some(Err(err)) => panic!("next frame failed: {err}"), + None => break, }, }; match reading.poll_read_chunk(&waiter) { diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index a5183c75f19..0541b755808 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -1657,8 +1657,7 @@ impl Consumer { let Some(expiry) = &self.expiry else { return false; }; - let budget = self.recover.as_ref().and_then(|recover| recover.0.poll_budget(waiter)); - if !expiry.policy.is_expired(budget, waiter) { + if !expiry.policy.is_expired(self.recover.as_ref().map(|recover| &recover.0), waiter) { return false; } diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index db4aca9b9a9..d4365ca4d2f 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -3385,8 +3385,9 @@ pub(super) trait Expiry: Send + Sync + std::panic::UnwindSafe + std::panic::RefU /// Return whether the group is stale, registering `waiter` for anything that /// could change the answer while the group remains live. - /// A logical reader supplies its current budget after the original copy is gone. - fn is_expired(&self, max_delay: Option, waiter: &kio::Waiter) -> bool; + /// A logical reader's `recover` supplies its current budget after the original copy is + /// gone, asked only once a verdict needs it. + fn is_expired(&self, recover: Option<&super::resume::Recover>, waiter: &kio::Waiter) -> bool; /// Keep the reader's budget and cap while following a replacement track's edge. fn for_track(&self, track: &track::Consumer) -> Arc; @@ -3700,10 +3701,11 @@ impl Consumer { fn poll_expiry(&mut self, waiter: &kio::Waiter, held: bool) -> bool { if !self.expired && (held || self.expiry_pending()) - && self.expiry.as_ref().is_some_and(|expiry| { - let budget = self.recover.as_ref().and_then(|recover| recover.poll_budget(waiter)); - expiry.is_expired(budget, waiter) - }) { + && self + .expiry + .as_ref() + .is_some_and(|expiry| expiry.is_expired(self.recover.as_deref(), waiter)) + { self.expired = true; // Another read, such as `poll_expired`, may judge a frame read parked before it. Idle::disarm(&mut self.idle); @@ -3813,7 +3815,7 @@ impl Consumer { /// /// Returns None if the group is finished and the index is out of range, or the cursor /// passed the [`Self::with_range`] cap. - pub fn poll_next_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { + pub(crate) fn poll_next_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { let res = match self.recover.is_none() { true => { let res = self.poll_next_frame_once(waiter); @@ -6676,7 +6678,7 @@ mod test { } impl Expiry for Switch { - fn is_expired(&self, _: Option, waiter: &kio::Waiter) -> bool { + fn is_expired(&self, _: Option<&crate::model::resume::Recover>, waiter: &kio::Waiter) -> bool { waiter.register(&mut self.0.1.lock().unwrap()); self.0.0.load(Ordering::Relaxed) } diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 0baf8b7f886..1c95cd55316 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1752,6 +1752,11 @@ impl Reader { for run in covered { self.drops.remove(run); } + // The common case: nothing to report, and the serving copy's own poll above wakes + // this for the next run it gives up, so no copy needs judging. + if self.drops.is_empty() { + return Poll::Pending; + } // Judged as the group path judges it, so a copy kept only for a group still read // holds nothing back, and letting one go wakes this too. self.let_go(delivered, waiter); @@ -6827,7 +6832,6 @@ mod test { assert!(sub.recv_group().now_or_never().is_none()); assert!(a.subscription().is_none()); } - wake.0.store(false, std::sync::atomic::Ordering::Relaxed); sub.control() .update( Subscription::default() @@ -6835,9 +6839,13 @@ mod test { .with_delay(Delay::default().with_max(Duration::from_secs(if widen { 10 } else { 2 }))), ) .unwrap(); - assert!( - wake.0.load(std::sync::atomic::Ordering::Relaxed), - "recovered={recovered}, widen={widen}, chunked={chunked}: preference update did not wake the held read" + // The update reaches the copy being read at once. Nothing landed above the + // held group, so no budget can convict it yet: the read need not wake until + // a group lands, and judges the new budget then. + assert_eq!( + source.subscription().unwrap().priority, + 7, + "recovered={recovered}, widen={widen}, chunked={chunked}: preference update did not reach the copy" ); for sequence in 1..=4 { let mut next = source.create_group(group::Info { sequence }).unwrap(); diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 455082fed03..58f60def6a7 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -5934,11 +5934,12 @@ impl group::Expiry for FetchExpiry { self.reading.release(); } - fn is_expired(&self, max_delay: Option, waiter: &kio::Waiter) -> bool { + fn is_expired(&self, recover: Option<&super::resume::Recover>, waiter: &kio::Waiter) -> bool { // Parked first, so a group landing above while this judges wakes it. if !self.cache.wakes().watch_landing(self.sequence, waiter) { return false; } + let max_delay = recover.and_then(|recover| recover.poll_budget(waiter)); let state = self.state.read(); let budget = match (self.max_delay, max_delay) { (Some(fetch), Some(route)) => fetch.min(route), @@ -5978,14 +5979,16 @@ impl group::Expiry for GroupExpiry { } /// A group nothing ever landed above has no successor, so neither clock can convict it: - /// that verdict, the common one at the live edge, takes neither the track's lock nor the - /// subscription's. Nothing it skips parking on can move it until a group lands above, - /// which wakes the read. - fn is_expired(&self, max_delay: Option, waiter: &kio::Waiter) -> bool { + /// that verdict, the common one at the live edge, takes neither the track's lock, the + /// subscription's, nor a logical reader's. Nothing it skips parking on can move it until + /// a group lands above, which wakes the read; a reader's preferences reach its copies + /// without this poll, since every update mirrors them at once. + fn is_expired(&self, recover: Option<&super::resume::Recover>, waiter: &kio::Waiter) -> bool { // Parked first, so a group landing above while this judges wakes it. if !self.cache.wakes().watch_landing(self.sequence, waiter) { return false; } + let max_delay = recover.and_then(|recover| recover.poll_budget(waiter)); let max_delay = max_delay.unwrap_or_else(|| { let mut max_delay = Duration::default(); let _ = self.subscription.poll(waiter, |subscription| { From f2d552cca7370cb9ba3fc636ed2d03f66c4d3559 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 22:27:29 -0800 Subject: [PATCH 08/16] perf(net): a group read serves its prefetch first; a post-receive drop poll skips the probe A prefetched frame is served before the idle gate and route bookkeeping that moq-dev/moq#7504 put on every frame read, and the read path below it has one call site again so it inlines. A relay draining a 512-frame group per reader (the `group` fan-out scenario, group_relay_frames/plain/512) drops from 173,221 to 135,396 instructions per group, below its 149,368 before that rise. The relay's track side polls drops again after a receive that parked, to collect a stale skip the receive's probe queued. That probe just judged the live edge with the same waiter, so the second poll now only collects. Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/lite/publisher.rs | 14 +++++-- rs/moq-net/src/model/group.rs | 66 ++++++++++++++++---------------- rs/moq-net/src/model/resume.rs | 17 ++++++-- rs/moq-net/src/model/track.rs | 23 +++++++++-- 4 files changed, 77 insertions(+), 43 deletions(-) diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 426bf81a649..61eed385253 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -5022,7 +5022,7 @@ impl TrackRun { writer: &mut Writer, waiter: &kio::Waiter, ) -> Poll, Error>> { - if self.poll_forward_drop(writer, waiter)? || self.report_started(false, writer)? { + if self.poll_forward_drop(true, writer, waiter)? || self.report_started(false, writer)? { return Poll::Ready(Ok(ControlFlow::Continue(()))); } @@ -5145,7 +5145,8 @@ impl TrackRun { } // Receiving can queue a stale skip after the poll above found nothing, without a wake. - if self.poll_forward_drop(writer, waiter)? { + // Its probe just judged the live edge, so this only collects what it queued. + if self.poll_forward_drop(false, writer, waiter)? { return Poll::Ready(Ok(ControlFlow::Continue(()))); } // A bounded subscription none of whose groups can be served has no group to resolve @@ -5171,15 +5172,22 @@ impl TrackRun { /// Lite-05+ names a sequence as soon as this cursor gives it up; whether one was taken. /// Before SUBSCRIBE_START there is no owed range yet, so the eventual start accounts /// for it instead, including ranges skipped while finding the first servable group. + /// Without `probe`, only collects runs a read already gave up; see + /// [`track::Subscriber::poll_queued_drop`]. fn poll_forward_drop( &mut self, + probe: bool, writer: &mut Writer, waiter: &kio::Waiter, ) -> Result { if !self.emit_range { return Ok(false); } - let Poll::Ready((range, err)) = self.track.poll_drop(waiter) else { + let polled = match probe { + true => self.track.poll_drop(waiter), + false => self.track.poll_queued_drop(waiter), + }; + let Poll::Ready((range, err)) = polled else { return Ok(false); }; match self.start { diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index d4365ca4d2f..852d5b2688a 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -3900,41 +3900,35 @@ impl Consumer { waiter: &kio::Waiter, take: impl Fn(&mut Prefetch) -> Option + Copy, ) -> Poll>> { + // Most reads of a backlog: kept off the read path below, whose bookkeeping costs + // every frame drained more than the pop. + if let Some(frame) = self.take_prefetched(take) { + return Poll::Ready(Ok(Some(frame))); + } if Idle::parked(&self.idle, (), waiter) { return Poll::Pending; } - let res = match self.idle.is_some() { - true => { - let mut idle = Idle::start(&mut self.idle, waiter); - let res = self.poll_read_frame_through(idle.waiter(), take); - idle.finish(&mut self.idle, (), res.is_pending()); - res - } - false => { - let res = self.poll_read_frame_through(waiter, take); - let parked = res.is_pending(); + let mut gate = self.idle.is_some().then(|| Idle::start(&mut self.idle, waiter)); + let read = gate.as_mut().map_or(waiter, |gate| gate.waiter()); + // Read once, carried across route changes when the group came from a front. One call + // site keeps the read inline: a second costs every frame drained a call frame. + let res = match self.recover.is_none() { + true => self.poll_read_frame_once(read, take), + false => self.poll_resumed(read, |this, waiter| this.poll_read_frame_once(waiter, take)), + }; + let parked = res.is_pending(); + match gate { + Some(gate) => gate.finish(&mut self.idle, (), parked), + None => { if parked && self.parked { // Parked twice in a row: the next read parks through a gate. Unarmed, since // this read registered on the task's own waiter, not the gate's. Idle::start(&mut self.idle, waiter).finish(&mut self.idle, (), false); } self.parked = parked; - res } - }; - self.ended_on(res) - } - - /// Read the next frame, carried across route changes when the group came from a front. - fn poll_read_frame_through( - &mut self, - waiter: &kio::Waiter, - take: impl Fn(&mut Prefetch) -> Option + Copy, - ) -> Poll>> { - match self.recover.is_none() { - true => self.poll_read_frame_once(waiter, take), - false => self.poll_resumed(waiter, |this, waiter| this.poll_read_frame_once(waiter, take)), } + self.ended_on(res) } /// Read the next frame with its relay metadata, for wire egress. @@ -3964,20 +3958,26 @@ impl Consumer { /// Read the next frame (timestamp and payload) all at once. #[cfg(feature = "async")] pub async fn read_frame(&mut self) -> Result> { - // A prefetched frame is already buffered, so the drift budget (which only judges - // a read that would park) can never apply to it. - // Serve from the prefetched batch without building a future or allocating a waker. - if !self.expired - && !self.cursor.capped() - && let Some(frame) = self.cursor.prefetch.pop() - { - self.cursor.refresh_if_stale(); - self.cursor.index += 1; + // Without building a future or allocating a waker. + if let Some(frame) = self.take_prefetched(Prefetch::pop) { return Ok(Some(frame)); } kio::wait(|waiter| self.poll_read_frame(waiter)).await } + /// The next frame from the prefetched batch, through `take`. A read parks only with its + /// batch drained, so neither the drift budget, which judges only a read that would park, + /// nor the idle gate, nor a route applies: one adopted onto a new copy starts empty. + fn take_prefetched(&mut self, take: impl Fn(&mut Prefetch) -> Option) -> Option { + if self.expired || self.cursor.capped() { + return None; + } + let frame = take(&mut self.cursor.prefetch)?; + self.cursor.refresh_if_stale(); + self.cursor.index += 1; + Some(frame) + } + /// Fill `out` with every frame that is ready, up to its capacity, without blocking. /// /// This is a short read: it returns as soon as anything is ready rather than diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 1c95cd55316..33ecb4d89e4 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -902,9 +902,10 @@ impl Subscriber { pub(super) fn poll_drop( &mut self, delivered: &Delivered, + probe: bool, waiter: &kio::Waiter, ) -> Poll<(std::ops::Range, StreamError)> { - self.reader().poll_drop(delivered, waiter) + self.reader().poll_drop(delivered, probe, waiter) } /// Assign the local read floor and exclusive cap; see [`track::Subscriber::with_range`]. @@ -1731,10 +1732,20 @@ impl Reader { /// through its [`Recover`]. A copy it replaced may still deliver what is under its bound, /// so a run there waits until that copy no longer feeds. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - fn poll_drop(&mut self, delivered: &Delivered, waiter: &kio::Waiter) -> Poll<(std::ops::Range, StreamError)> { + fn poll_drop( + &mut self, + delivered: &Delivered, + probe: bool, + waiter: &kio::Waiter, + ) -> Poll<(std::ops::Range, StreamError)> { let mut given_up = Vec::new(); if let Poll::Ready(Some(sub)) = self.poll_serving(waiter) { - while let Poll::Ready(run) = sub.poll_drop(waiter) { + loop { + let polled = match probe { + true => sub.poll_drop(waiter), + false => sub.poll_queued_drop(waiter), + }; + let Poll::Ready(run) = polled else { break }; given_up.push(run); } } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 58f60def6a7..6f93e0e360f 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -6772,8 +6772,9 @@ impl Cursor { } } + /// The next run given up, judging the live edge first only when `probe`. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - fn poll_drop(&mut self, taken: &Delivered, waiter: &kio::Waiter) -> Poll<(ops::Range, StreamError)> { + fn poll_drop(&mut self, taken: &Delivered, probe: bool, waiter: &kio::Waiter) -> Poll<(ops::Range, StreamError)> { self.sync(); if self.drops.is_none() { let mut drops = Box::new(DropCursor::new(&self.state.read())); @@ -6784,7 +6785,8 @@ impl Cursor { } self.drops = Some(drops); } - if self.filter.refused.is_none() + if probe + && self.filter.refused.is_none() && let Poll::Ready(Ok(drift)) = self.poll_drift(&self.servable, waiter) { // From where the reader reads next: an ordered one asks nothing below what it took. @@ -8655,9 +8657,22 @@ impl Subscriber { /// forwards drops keeps none. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn poll_drop(&mut self, waiter: &kio::Waiter) -> Poll<(ops::Range, StreamError)> { + self.poll_drop_probing(true, waiter) + } + + /// [`Self::poll_drop`] without judging the live edge again: only the runs a read since + /// the last judgement gave up, such as a receive that probed after a drop poll found + /// nothing. That judgement parked `waiter` on everything that could change it. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn poll_queued_drop(&mut self, waiter: &kio::Waiter) -> Poll<(ops::Range, StreamError)> { + self.poll_drop_probing(false, waiter) + } + + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + fn poll_drop_probing(&mut self, probe: bool, waiter: &kio::Waiter) -> Poll<(ops::Range, StreamError)> { match &mut self.inner { - Inner::Plain(cursor) => cursor.poll_drop(&self.delivered, waiter), - Inner::Resume(resume, _) => resume.poll_drop(&self.delivered, waiter), + Inner::Plain(cursor) => cursor.poll_drop(&self.delivered, probe, waiter), + Inner::Resume(resume, _) => resume.poll_drop(&self.delivered, probe, waiter), } } From 978f09a7d68f7dcac65ee56b80a6ecceb2e0f226 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 22:37:04 -0800 Subject: [PATCH 09/16] docs(bench): finish the relay CPU bisect: public ingest's live read and group_relay_frames Records the three session cuts and the group drain's attribution in the report and evidence, accepts what remains of public ingest's per-viewer read in the release-perf epic, updates the pacing quest with its counted cost, and deletes the finished quest. Co-Authored-By: Claude Opus 5.5 --- .github/relay-cpu-bisect.json | 548 +++++++++++++++++++++- .github/relay-cpu-bisect.md | 38 +- quest/m0/bench-host-lock.md | 4 - quest/m0/kio-waiter-records.md | 1 - quest/m1/release-perf/README.md | 24 +- quest/m1/release-perf/pacing-report.md | 14 +- quest/m1/release-perf/relay-cpu-bisect.md | 32 -- 7 files changed, 615 insertions(+), 46 deletions(-) delete mode 100644 quest/m1/release-perf/relay-cpu-bisect.md diff --git a/.github/relay-cpu-bisect.json b/.github/relay-cpu-bisect.json index ab7430b569c..0ac8ee20b57 100644 --- a/.github/relay-cpu-bisect.json +++ b/.github/relay-cpu-bisect.json @@ -3480,7 +3480,7 @@ ] } ], - "fanout_method": "just bench-fanout: user instructions per delivered group on one pinned thread, a run of many cycles less a run of zero, at the listed subscriber counts; model = the track model alone, session = publisher, relay origin, and viewers over the in-memory transport", + "fanout_method": "just bench-fanout: user instructions per delivered group on one pinned thread, a run of many cycles less a run of zero, at the listed subscriber counts; model = the track model alone, session = publisher, relay origin, and viewers over the in-memory transport; group = every subscriber drains one finished 512-frame group in 16-frame batches, as group_relay_frames/plain/512 does, per drained group", "fanout_runs": [ { "artifact": "hist", @@ -5368,6 +5368,552 @@ "per_group": 109123.37050581952 } ] + }, + { + "artifact": "desktop:run.N037x9", + "note": "the reader's budget asked only past the landing check (working tree) against this branch at 45351ac1d8 (base), with callgrind", + "harness_revision": "45351ac1d8025de6050557246b751f584e096a3a", + "frame_pointers": false, + "counts": [ + { + "label": "base", + "revision": "45351ac1d8025de6050557246b751f584e096a3a", + "scenario": "session", + "subscribers": 1, + "per_group": 225948.00391 + }, + { + "label": "base", + "revision": "45351ac1d8025de6050557246b751f584e096a3a", + "scenario": "session", + "subscribers": 512, + "per_group": 110581.19509327169 + }, + { + "label": "current", + "revision": "45351ac1d8025de6050557246b751f584e096a3a", + "scenario": "session", + "subscribers": 1, + "per_group": 224883.27597 + }, + { + "label": "current", + "revision": "45351ac1d8025de6050557246b751f584e096a3a", + "scenario": "session", + "subscribers": 512, + "per_group": 109968.52706473214 + } + ], + "callgrind": { + "callgrind program total, session 512": { + "base": 105677.4, + "current": 105084.2 + } + } + }, + { + "artifact": "desktop:run.vS3CAR", + "note": "the lazy budget plus an empty drop report returning at once (working tree) against 45351ac1d8, with callgrind", + "harness_revision": "45351ac1d8025de6050557246b751f584e096a3a", + "frame_pointers": false, + "counts": [ + { + "label": "base", + "revision": "45351ac1d8025de6050557246b751f584e096a3a", + "scenario": "session", + "subscribers": 512, + "per_group": 110582.39753069196 + }, + { + "label": "current", + "revision": "45351ac1d8025de6050557246b751f584e096a3a", + "scenario": "session", + "subscribers": 512, + "per_group": 108802.951430963 + } + ], + "callgrind": { + "callgrind program total, session 512": { + "base": 105676.3, + "current": 103945.7 + } + } + }, + { + "artifact": "desktop:run.yLeXzU", + "note": "group drain at the quest endpoints e7a5c10913 and f37d964dd3, and pacing (moq-dev/moq#7388, 7ad8d426e0) against its parent b51eb8cdec; current is c6753bd267 plus the group harness", + "harness_revision": "45351ac1d8025de6050557246b751f584e096a3a", + "frame_pointers": false, + "counts": [ + { + "label": "base-e7a5c10913b0", + "revision": "e7a5c10913b02e21ef851a0250b594fe837d1425", + "scenario": "group", + "subscribers": 1, + "per_group": 149368.3935 + }, + { + "label": "base-e7a5c10913b0", + "revision": "e7a5c10913b02e21ef851a0250b594fe837d1425", + "scenario": "group", + "subscribers": 64, + "per_group": 149362.45849609375 + }, + { + "label": "base-b51eb8cdecc3", + "revision": "b51eb8cdecc33c858f6df8f3b2ccbcba88b9535e", + "scenario": "group", + "subscribers": 1, + "per_group": 149368.3935 + }, + { + "label": "base-b51eb8cdecc3", + "revision": "b51eb8cdecc33c858f6df8f3b2ccbcba88b9535e", + "scenario": "group", + "subscribers": 64, + "per_group": 149362.45947265625 + }, + { + "label": "base-7ad8d426e072", + "revision": "7ad8d426e072a4b1f03483204ec2c9ae8d3d3ffb", + "scenario": "group", + "subscribers": 1, + "per_group": 152375.13 + }, + { + "label": "base-7ad8d426e072", + "revision": "7ad8d426e072a4b1f03483204ec2c9ae8d3d3ffb", + "scenario": "group", + "subscribers": 64, + "per_group": 152369.22412109375 + }, + { + "label": "base-f37d964dd319", + "revision": "f37d964dd31987efe34b55375902de1380474235", + "scenario": "group", + "subscribers": 1, + "per_group": 172205.171 + }, + { + "label": "base-f37d964dd319", + "revision": "f37d964dd31987efe34b55375902de1380474235", + "scenario": "group", + "subscribers": 64, + "per_group": 172199.2099609375 + }, + { + "label": "current", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "group", + "subscribers": 1, + "per_group": 173221.225 + }, + { + "label": "current", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "group", + "subscribers": 64, + "per_group": 173215.2705078125 + } + ] + }, + { + "artifact": "desktop:run.S9A3hY", + "note": "group drain after each candidate commit between pacing and f37d964dd3: the step is moq-dev/moq#7504 (25ff6e9754 against ad3ca7784a), then moq-dev/moq#7791 (8b46ba5b33)", + "harness_revision": "45351ac1d8025de6050557246b751f584e096a3a", + "frame_pointers": false, + "counts": [ + { + "label": "base-2ec2f1d55e55", + "revision": "2ec2f1d55e5588f32374f5b004a7d6bd7efc9453", + "scenario": "group", + "subscribers": 1, + "per_group": 152375.3145 + }, + { + "label": "base-dde50c699f69", + "revision": "dde50c699f69507ced243754c2f700a6184d1576", + "scenario": "group", + "subscribers": 1, + "per_group": 152375.3475 + }, + { + "label": "base-24b04633a58c", + "revision": "24b04633a58cbff0144531efdb450cf227133e67", + "scenario": "group", + "subscribers": 1, + "per_group": 152375.134 + }, + { + "label": "base-f1034b7cc973", + "revision": "f1034b7cc973009bb33abfd06404be1a1dea0266", + "scenario": "group", + "subscribers": 1, + "per_group": 152375.1585 + }, + { + "label": "base-5ab878666c31", + "revision": "5ab878666c312fc2a497a31656630efeb02ca55f", + "scenario": "group", + "subscribers": 1, + "per_group": 152375.2045 + }, + { + "label": "base-ad3ca7784aa0", + "revision": "ad3ca7784aa00f7b8b93b8a8076304673c50e733", + "scenario": "group", + "subscribers": 1, + "per_group": 152375.2585 + }, + { + "label": "base-25ff6e97546f", + "revision": "25ff6e97546f59dcebde66f43b67ba55f590867c", + "scenario": "group", + "subscribers": 1, + "per_group": 171694.186 + }, + { + "label": "base-976fece17349", + "revision": "976fece17349666bb2ab6dad93109408ac272e8d", + "scenario": "group", + "subscribers": 1, + "per_group": 171694.1055 + }, + { + "label": "base-ee8e42a95b6c", + "revision": "ee8e42a95b6c2d880dc1fd0b846c47886330ecf7", + "scenario": "group", + "subscribers": 1, + "per_group": 171694.1265 + }, + { + "label": "base-8b46ba5b330f", + "revision": "8b46ba5b330f6bbc93aa415af6c27dbb935c4e6a", + "scenario": "group", + "subscribers": 1, + "per_group": 172205.6565 + }, + { + "label": "base-ffcb7cd97484", + "revision": "ffcb7cd9748422daa30e935ec464108b5a8ebb71", + "scenario": "group", + "subscribers": 1, + "per_group": 172205.268 + }, + { + "label": "current", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "group", + "subscribers": 1, + "per_group": 173221.276 + } + ] + }, + { + "artifact": "desktop:run.R849HM", + "note": "group drain with callgrind across moq-dev/moq#7504", + "harness_revision": "45351ac1d8025de6050557246b751f584e096a3a", + "frame_pointers": false, + "counts": [ + { + "label": "base-ad3ca7784aa0", + "revision": "ad3ca7784aa00f7b8b93b8a8076304673c50e733", + "scenario": "group", + "subscribers": 1, + "per_group": 152375.0475 + }, + { + "label": "base-25ff6e97546f", + "revision": "25ff6e97546f59dcebde66f43b67ba55f590867c", + "scenario": "group", + "subscribers": 1, + "per_group": 171694.049 + }, + { + "label": "current", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "group", + "subscribers": 1, + "per_group": 173221.221 + } + ], + "callgrind": { + "callgrind group 1, 25ff6e9754 against ad3ca7784a": { + "program total": 19319.0, + "poll_read_frame_through, new out-of-line layer, self": 10773.0, + "poll_read_frame self": 7182.0, + "Park::hold self": 1452.0 + } + } + }, + { + "artifact": "desktop:run.DV13e7", + "note": "one inline read call site plus the drain-only post-receive drop poll (working tree) against c6753bd267", + "harness_revision": "45351ac1d8025de6050557246b751f584e096a3a", + "frame_pointers": false, + "counts": [ + { + "label": "base", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "group", + "subscribers": 1, + "per_group": 173221.169 + }, + { + "label": "base", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "group", + "subscribers": 512, + "per_group": 173215.162109375 + }, + { + "label": "current", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "group", + "subscribers": 1, + "per_group": 169115.225 + }, + { + "label": "current", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "group", + "subscribers": 512, + "per_group": 169109.18017578125 + }, + { + "label": "base", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "session", + "subscribers": 1, + "per_group": 222587.67073 + }, + { + "label": "base", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "session", + "subscribers": 512, + "per_group": 108824.46070830675 + }, + { + "label": "current", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "session", + "subscribers": 1, + "per_group": 218679.16904 + }, + { + "label": "current", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "session", + "subscribers": 512, + "per_group": 106832.64318598533 + } + ] + }, + { + "artifact": "desktop:run.lr8Dpl", + "note": "the prefetch served first, with the changes above (working tree), against c6753bd267 and ad3ca7784a (before moq-dev/moq#7504)", + "harness_revision": "45351ac1d8025de6050557246b751f584e096a3a", + "frame_pointers": false, + "counts": [ + { + "label": "base-c6753bd267f8", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "group", + "subscribers": 1, + "per_group": 173221.166 + }, + { + "label": "base-c6753bd267f8", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "group", + "subscribers": 512, + "per_group": 173215.119140625 + }, + { + "label": "base-ad3ca7784aa0", + "revision": "ad3ca7784aa00f7b8b93b8a8076304673c50e733", + "scenario": "group", + "subscribers": 1, + "per_group": 152375.051 + }, + { + "label": "base-ad3ca7784aa0", + "revision": "ad3ca7784aa00f7b8b93b8a8076304673c50e733", + "scenario": "group", + "subscribers": 512, + "per_group": 152369.01806640625 + }, + { + "label": "current", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "group", + "subscribers": 1, + "per_group": 135396.2205 + }, + { + "label": "current", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "group", + "subscribers": 512, + "per_group": 135390.17578125 + }, + { + "label": "base-c6753bd267f8", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "session", + "subscribers": 1, + "per_group": 222589.22774 + }, + { + "label": "base-c6753bd267f8", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "session", + "subscribers": 512, + "per_group": 108795.25971580038 + }, + { + "label": "base-ad3ca7784aa0", + "revision": "ad3ca7784aa00f7b8b93b8a8076304673c50e733", + "scenario": "session", + "subscribers": 1, + "per_group": 243706.45894 + }, + { + "label": "base-ad3ca7784aa0", + "revision": "ad3ca7784aa00f7b8b93b8a8076304673c50e733", + "scenario": "session", + "subscribers": 512, + "per_group": 120849.10758131377 + }, + { + "label": "current", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "session", + "subscribers": 1, + "per_group": 218664.50306 + }, + { + "label": "current", + "revision": "c6753bd267f892c62ec1a2b8bff25ab559a68d8a", + "scenario": "session", + "subscribers": 512, + "per_group": 106852.6315369898 + } + ] + }, + { + "artifact": "desktop:run.1gKfjA", + "note": "this PR (f2d552cca7) against 45351ac1d8 and e7a5c10913, with callgrind", + "harness_revision": "f2d552cca7", + "frame_pointers": false, + "counts": [ + { + "label": "base-45351ac1d802", + "revision": "45351ac1d8025de6050557246b751f584e096a3a", + "scenario": "group", + "subscribers": 1, + "per_group": 173231.1615 + }, + { + "label": "base-45351ac1d802", + "revision": "45351ac1d8025de6050557246b751f584e096a3a", + "scenario": "group", + "subscribers": 512, + "per_group": 173225.1162109375 + }, + { + "label": "base-e7a5c10913b0", + "revision": "e7a5c10913b02e21ef851a0250b594fe837d1425", + "scenario": "group", + "subscribers": 1, + "per_group": 149368.395 + }, + { + "label": "base-e7a5c10913b0", + "revision": "e7a5c10913b02e21ef851a0250b594fe837d1425", + "scenario": "group", + "subscribers": 512, + "per_group": 149362.369140625 + }, + { + "label": "current", + "revision": "f2d552cca7370cb9ba3fc636ed2d03f66c4d3559", + "scenario": "group", + "subscribers": 1, + "per_group": 135396.222 + }, + { + "label": "current", + "revision": "f2d552cca7370cb9ba3fc636ed2d03f66c4d3559", + "scenario": "group", + "subscribers": 512, + "per_group": 135390.17578125 + }, + { + "label": "base-45351ac1d802", + "revision": "45351ac1d8025de6050557246b751f584e096a3a", + "scenario": "session", + "subscribers": 1, + "per_group": 225948.01262 + }, + { + "label": "base-45351ac1d802", + "revision": "45351ac1d8025de6050557246b751f584e096a3a", + "scenario": "session", + "subscribers": 512, + "per_group": 110581.04552973533 + }, + { + "label": "base-e7a5c10913b0", + "revision": "e7a5c10913b02e21ef851a0250b594fe837d1425", + "scenario": "session", + "subscribers": 1, + "per_group": 233562.49815 + }, + { + "label": "base-e7a5c10913b0", + "revision": "e7a5c10913b02e21ef851a0250b594fe837d1425", + "scenario": "session", + "subscribers": 512, + "per_group": 115444.22222775829 + }, + { + "label": "current", + "revision": "f2d552cca7370cb9ba3fc636ed2d03f66c4d3559", + "scenario": "session", + "subscribers": 1, + "per_group": 218664.30033 + }, + { + "label": "current", + "revision": "f2d552cca7370cb9ba3fc636ed2d03f66c4d3559", + "scenario": "session", + "subscribers": 512, + "per_group": 106852.35546875 + } + ], + "callgrind": { + "callgrind program total": { + "session 512": { + "base 45351ac1d8": 105664.8, + "current": 101955.4 + }, + "session 1": { + "base 45351ac1d8": 224392.4, + "current": 217119.1 + }, + "group 1": { + "base 45351ac1d8": 173254.5, + "current": 135420.1 + } + }, + "current session 512 by caller": { + "Cursor::probe from the drop poll": 3467, + "Cursor::probe from the receive peek": 3460, + "group::Consumer::poll_expiry (a blocked read and an opening group)": 886 + } + } } ], "quiet_host_runs": [ diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md index b7ea11130d6..48c87f8f5d1 100644 --- a/.github/relay-cpu-bisect.md +++ b/.github/relay-cpu-bisect.md @@ -160,7 +160,43 @@ Neither costs measurably: each change is inside its own rounds' spread. The rise Public ingest added +10.5k per frame here, against +200 per group through the model, because the model scenario never reads a frame. It stopped withholding groups: every group joins arrival order when created, so each viewer's read reaches a group before its first frame and blocks on it. By function (rounds 2 and 3), the rise is each viewer's group serving (`GroupServe`, +5.3k inclusive: `group::Consumer::poll_read_frame` and the new `poll_next_frame`, with the expiry checks a blocked read runs, `poll_expired_if_blocked`, `poll_expired_while_pending`, `Recover::poll_budget`, and `watch_landing`), new track-state lock polls in the forward-drop, receive, and staleness paths (+3.8k), and the lock and unlock leaves (+2.0k self). -On `main`, the in-process session's group read (`group::Consumer::poll_read_objects`) costs 4.5k per delivered group (run 4rAIYS), of which `GroupExpiry::is_expired` is 1.6k, `Consumer::poll_expiry` 1.4k, and `Recover::poll_budget` 1.0k. `poll_expiry` asks the budget before `is_expired`, which answers false without it while nothing landed above the group, but the budget's poll also syncs the reader's mirrored preferences, so skipping it is not a change that can be made without proving that sync redundant. +## Public ingest's per-viewer read on `main` + +Decided 2026-11-05 (maintainer): keep cutting public ingest's relay cost. The in-process session reads every frame, so it reaches the read path public ingest opened. Counted at 512 subscribers per delivered group, each change against the commit before it: + +| Change | Session | Change | +| --- | ---: | ---: | +| This branch at [45351ac1d8](https://github.com/moq-dev/moq/commit/45351ac1d8) (`main` at [246916845a](https://github.com/moq-dev/moq/commit/246916845a)) | 110,581 | | +| A blocked read asks the reader's budget only once a group landed above it | 109,968 | -613 | +| A reader's drop poll with nothing to report returns at once | 108,803 | -1,166 | +| A drop poll right after a receive only collects what the receive queued, and the frame read inlines again | 106,833 | -1,992 | +| A prefetched frame is read before the idle gate | 106,853 | 0 | +| All of them (run 1gKfjA; callgrind 105,665 to 101,955) | 106,852 | -3,729 (-3.4%) | + +Each change is counted against the row above in its own run (N037x9, vS3CAR, DV13e7, lr8Dpl), and the total in one run. A chat group has one frame, so the prefetch change shows only in the `group` drain below. + +- **The budget.** A read that blocks before its group's first frame judges the group's expiry. `Consumer::poll_expiry` asked the logical reader's budget (`Recover::poll_budget`: the reader's lock and a sync of its mirrored preferences) before `is_expired`, which answers false without the budget until a group lands above. The sync is redundant there: a preference update mirrors into every copy at once (`Mirror::sync`), a route change syncs every reader (`serve_from`), and the landing that makes the budget matter wakes the read through `watch_landing`, which then asks the budget. `is_expired` now takes the `Recover` and asks only past that check. +- **The forward drop.** The relay's track side polls the reader for runs to report (`TrackRun::poll_forward_drop`) on each track wake, and again after a receive that parked, to collect a stale skip the receive's probe queued without a wake. Each poll judged every copy (`Reader::let_go`) with nothing to report, and the second probed the live edge again (`Cursor::poll_drift` and `probe`) right after the receive had, with the same waiter. The reader now returns once nothing is queued, and the second poll only collects. Drop probes fall from 5 to 3 per delivered group. + +What remains of each viewer's live read is accepted in the [release-perf epic](../quest/m1/release-perf/README.md): about 3.5k per delivered group in the drop poll's probe and 3.5k in the receive's (three each, one wake of each kind), and 0.9k in the expiry checks of a read that blocks before its first frame and of a group waiting for stream credit, which park on the next landing. + +## `group_relay_frames` since `e7a5c10` + +[Fastest-reader pacing](https://github.com/moq-dev/moq/pull/7664)'s comparison found `group_relay_frames` about 15% slower at [f37d964dd3](https://github.com/moq-dev/moq/commit/f37d964dd3) than at [e7a5c10913](https://github.com/moq-dev/moq/commit/e7a5c10913) on a loaded host. The `group` fan-out scenario counts the same drain (every subscriber reads a finished 512-frame group in 16-frame batches), per drained group (runs yLeXzU, S9A3hY, R849HM): + +| Revision | Instructions | Change | +| --- | ---: | ---: | +| [e7a5c10913](https://github.com/moq-dev/moq/commit/e7a5c10913) | 149,368 | | +| [b51eb8cdec](https://github.com/moq-dev/moq/commit/b51eb8cdec), before pacing | 149,368 | | +| [7ad8d426e0](https://github.com/moq-dev/moq/commit/7ad8d426e0), pacing, [moq-dev/moq#7388](https://github.com/moq-dev/moq/pull/7388) | 152,375 | +3,007 (+2.0%) | +| [ad3ca7784a](https://github.com/moq-dev/moq/commit/ad3ca7784a), before idle frame reads | 152,375 | | +| [25ff6e9754](https://github.com/moq-dev/moq/commit/25ff6e9754), idle frame reads, [moq-dev/moq#7504](https://github.com/moq-dev/moq/pull/7504) | 171,694 | +19,319 (+12.7%) | +| [8b46ba5b33](https://github.com/moq-dev/moq/commit/8b46ba5b33), smaller group consumers, [moq-dev/moq#7791](https://github.com/moq-dev/moq/pull/7791) | 172,205 | +511 | +| [f37d964dd3](https://github.com/moq-dev/moq/commit/f37d964dd3) | 172,205 | | +| This branch at [45351ac1d8](https://github.com/moq-dev/moq/commit/45351ac1d8) | 173,231 | +1,026 | +| This PR (run 1gKfjA) | 135,396 | -37,835 (-21.8%) | + +The +15.3% matches the wall-clock report. Nine other candidates between pacing and `f37d964dd3` count the same as their predecessor. [moq-dev/moq#7504](https://github.com/moq-dev/moq/pull/7504) put an idle gate in front of every frame read and split the read into a second out-of-line layer called from two branches, so each frame paid a call frame and the gate's bookkeeping, about 38 instructions on top of about 297. A frame already in the reader's prefetched batch now returns before the gate, the drift budget, or a route, none of which can apply to it (a read parks only with its batch drained, and a copy adopted mid-group starts empty); what remains below has one call site. The drain is now 9.4% below `e7a5c10913`, pacing included. ## Validation diff --git a/quest/m0/bench-host-lock.md b/quest/m0/bench-host-lock.md index 97ef54343cb..3efe652cf33 100644 --- a/quest/m0/bench-host-lock.md +++ b/quest/m0/bench-host-lock.md @@ -48,7 +48,3 @@ Either way: - Nightly's hosted runners run one job per machine, so they need no change. Public API: none. Wire: none. - -## Related - -- [Find and fix the relay's per-frame CPU growth since #5926](/quest/m1/release-perf/relay-cpu-bisect.md) - the bisect that overlapped diff --git a/quest/m0/kio-waiter-records.md b/quest/m0/kio-waiter-records.md index f4225dae312..0aba6152558 100644 --- a/quest/m0/kio-waiter-records.md +++ b/quest/m0/kio-waiter-records.md @@ -31,5 +31,4 @@ Public API: none expected. Wire: none. ## Related -- [Relay CPU per frame since #5926](/quest/m1/release-perf/relay-cpu-bisect.md) - found this, measuring the waiter's registration cost - [Task sets release memory after a burst](/quest/m2/performance-survey/task-capacity.md) - a task slot's `Park` holds a whole `Waiter`, which 16 records grow diff --git a/quest/m1/release-perf/README.md b/quest/m1/release-perf/README.md index 70309d53862..6198a3e4332 100644 --- a/quest/m1/release-perf/README.md +++ b/quest/m1/release-perf/README.md @@ -333,13 +333,29 @@ per delivered group at 512 subscribers through the in-process session: In-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)) add no measurable relay cost on chat over io\_uring, each against its parent -on the quiet test host, so neither needs accepting. Public ingest's relay cost -is larger than the model counted and waits on a decision in the -[bisect](/quest/m1/release-perf/relay-cpu-bisect.md). +on the quiet test host, so neither needs accepting. + +Public ingest ([moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111)) +cost the relay +10.5k per frame on chat over io\_uring, far above the model's ++200, because each viewer's read now reaches a group before its first frame. +Decided 2026-11-05 (maintainer): cut what a live read pays for nothing. +[moq-dev/moq#7874](https://github.com/moq-dev/moq/pull/7874) took 3,729 per +delivered group off the in-process session at 512 subscribers (110,581 to +106,852, -3.4%): a blocked read asks the reader's budget only once a group +landed above it, and the forward-drop poll skips judging copies with nothing +to report and re-probing right after a receive. What remains of each viewer's +live read is accepted, per delivered group through the session (see the +[report](/.github/relay-cpu-bisect.md#public-ingests-per-viewer-read-on-main)): + +- The forward-drop poll's probe, about 3.5k over three polls a group, and the + receive's, about 3.5k over three: each wake of the track side checks for + groups the reader missed before taking or parking. +- The expiry checks of a read blocked before its group's first frame and of a + group waiting for stream credit, 0.9k: each parks on the next landing so a + stalled group expires. ## Required -- [Relay CPU per frame since #5926](/quest/m1/release-perf/relay-cpu-bisect.md) - the 18-56% rise in relay instructions per delivered frame is bisected, attributed, and its avoidable cost fixed - [Pacing's per-frame report](/quest/m1/release-perf/pacing-report.md) - relay fan-out reads with pacing are within noise of the pre-pacing parent ## Related diff --git a/quest/m1/release-perf/pacing-report.md b/quest/m1/release-perf/pacing-report.md index 86fafc27d59..009427056f4 100644 --- a/quest/m1/release-perf/pacing-report.md +++ b/quest/m1/release-perf/pacing-report.md @@ -17,6 +17,14 @@ pacing, against its parent `b51eb8c` is 4.6-8.1% slower on `group_relay_frames` in all 12 cases with 512+ frames (64-byte payloads, the worst case for a per-frame cost); 4-frame groups and `group_read_frames` are unchanged. The likely cause is the per-read progress report #7388 added to -the batch read path; profile it first (`just bench-profile instructions`, as -[the relay CPU bisect](/quest/m1/release-perf/relay-cpu-bisect.md) does), -then report progress per batch or only when the pacing window can change. +the batch read path; profile it first, then report progress per batch or +only when the pacing window can change. + +Counted 2026-11-05 in [moq-dev/moq#7874](https://github.com/moq-dev/moq/pull/7874) +with `just bench-fanout group`, which drains a 512-frame group as +`group_relay_frames/plain/512` does: pacing adds 3,007 instructions per drained +group (149,368 to 152,375, +2.0%). Most of the loaded host's 15% from +`e7a5c10` to `f37d964` was [moq-dev/moq#7504](https://github.com/moq-dev/moq/pull/7504)'s +idle gate on every frame read (+12.7%), which that PR took back with more: the +drain is now 135,396, 9.4% below pacing's parent, pacing included (see the +[report](/.github/relay-cpu-bisect.md#group_relay_frames-since-e7a5c10)). diff --git a/quest/m1/release-perf/relay-cpu-bisect.md b/quest/m1/release-perf/relay-cpu-bisect.md deleted file mode 100644 index 38ffe428a09..00000000000 --- a/quest/m1/release-perf/relay-cpu-bisect.md +++ /dev/null @@ -1,32 +0,0 @@ -# [M] Find and fix the relay's per-frame CPU growth - -## Goal - -The commits that raised the relay's instructions per delivered frame between [cdff3c8f95](https://github.com/moq-dev/moq/commit/cdff3c8f952b1d6bd2fd40c3f64eb0fd53b97e95) ([moq-dev/moq#5926](https://github.com/moq-dev/moq/pull/5926) as merged) and `main` are found, each rise is attributed to a cause, and the avoidable costs are fixed. A rise that pays for a deliberate feature is accepted with its reason in the [release-perf epic](/quest/m1/release-perf/README.md). - -## Plan - -Found by [moq-dev/moq#7505](https://github.com/moq-dev/moq/pull/7505). Decided 2026-11-01 (maintainer): m1, in this epic, since `main` must not reach the cut carrying accidental per-frame cost. - -[The report](/.github/relay-cpu-bisect.md) has every step, its cause, and its evidence. [moq-dev/moq#7605](https://github.com/moq-dev/moq/pull/7605) bisected the rise, fixed the accidental `Cursor::probe` work, and recorded the costs the maintainer accepted on 2026-11-03 in the epic. [moq-dev/moq#7874](https://github.com/moq-dev/moq/pull/7874) finished the remaining measurements: - -- [moq-dev/moq#6995](https://github.com/moq-dev/moq/pull/6995)'s and [moq-dev/moq#7542](https://github.com/moq-dev/moq/pull/7542)'s session costs were never the probe's judgement, so the fast path left both. One branch per live walk took back 891 per group; the rest is accepted in the epic. -- In-tree qmux ([moq-dev/moq#6231](https://github.com/moq-dev/moq/pull/6231)) and receive leases ([moq-dev/moq#5878](https://github.com/moq-dev/moq/pull/5878)) cost nothing measurable on the relay. The step between them is public ingest ([moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111)): +10.5k per frame on chat over io\_uring, in each viewer's frame read. -- `main` against the old endpoint, on the quiet test host: chat 1.28x on io\_uring and 1.10x on the tokio workers, video 1.12x and 0.89x. - -What remains is public ingest's relay cost and the `group_relay_frames` item below. The epic accepted public ingest on 2026-11-03 at its model cost (+200 per group), which never reads a frame. - -Decided 2026-11-05 (maintainer, /quest-unblock): keep cutting public ingest's relay cost ([moq-dev/moq#6111](https://github.com/moq-dev/moq/pull/6111), +10.5k of `main`'s +30.8k per frame on chat over io\_uring). Count `main`'s per-viewer read with `just bench-fanout` session and callgrind, fix what a live read pays for nothing, and record the rest in the epic. The listed lead is `Consumer::poll_expiry` calling `Recover::poll_budget` before `is_expired`; skipping it needs the mirrored-preference sync proven redundant. Not chosen: accepting the cost, or handing it to the [performance audit](/quest/m2/audit-perf.md). Also remove, or make crate-private, any public API on the relay read paths this work touches that no consumer outside its own crate uses. - -Measure as the report does. For a model or session change, `just bench-fanout` against the base, which host load doesn't move; the session scenario reads every frame, so it reaches public ingest's read path. For the relay, `BENCH_GENERATOR=21d6da9fc1`, `BENCH_INSTRUCTION_PERIOD=2000003`, `kernel.perf_event_max_sample_rate` raised to 100,000 and restored after, on a quiet host such as `nixos2` (relay on CPUs 0-3, generator on 4,5,10,11, `BENCH_BUILD_JOBS=2`), comparing only cells from one run. - -Added 2026-11-04 (maintainer) from [fastest-reader pacing](https://github.com/moq-dev/moq/pull/7664)'s comparison on a loaded host: `main` at `f37d964` reads `group_relay_frames` about 15% slower than `e7a5c10`. Pacing ([moq-dev/moq#7388](https://github.com/moq-dev/moq/pull/7388)) explains about 4% of that, and [its own quest](/quest/m1/release-perf/pacing-report.md) trims it; attribute the remaining ~10% here, on a quiet host. - -Public API: none expected. Wire: none. - -## Related - -- [A waiter's record limit changes no behavior](/quest/m0/kio-waiter-records.md) - found here; the session's waiter registration cost waits on it -- [Benchmark sessions can't overlap on a host](/quest/m0/bench-host-lock.md) - the quiet-host runs here were voided twice by other sessions' work -- [Calibrate the paired nightly benchmarks](/quest/m1/bench-ci.md) - the nightly series that should have caught this growth -- [Performance audit](/quest/m2/audit-perf.md) - counts instructions when bisecting, for the same reason From 8fa7235a5724f5c3a16e684becebcf3d2ed762fd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 22:48:35 -0800 Subject: [PATCH 10/16] style(net): rustfmt the drop poll and frame expiry Co-Authored-By: Claude Opus 5.5 --- rs/moq-net/src/model/frame.rs | 3 ++- rs/moq-net/src/model/track.rs | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index 0541b755808..5c080549ca5 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -1657,7 +1657,8 @@ impl Consumer { let Some(expiry) = &self.expiry else { return false; }; - if !expiry.policy.is_expired(self.recover.as_ref().map(|recover| &recover.0), waiter) { + let recover = self.recover.as_ref().map(|recover| &recover.0); + if !expiry.policy.is_expired(recover, waiter) { return false; } diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 6f93e0e360f..bb2a3204916 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -6774,7 +6774,12 @@ impl Cursor { /// The next run given up, judging the live edge first only when `probe`. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - fn poll_drop(&mut self, taken: &Delivered, probe: bool, waiter: &kio::Waiter) -> Poll<(ops::Range, StreamError)> { + fn poll_drop( + &mut self, + taken: &Delivered, + probe: bool, + waiter: &kio::Waiter, + ) -> Poll<(ops::Range, StreamError)> { self.sync(); if self.drops.is_none() { let mut drops = Box::new(DropCursor::new(&self.state.read())); From bbdb30d989a32af9940d25187993d31915885436 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 08:58:50 -0800 Subject: [PATCH 11/16] docs(bench): distinguish historical frame batch measurements --- .github/relay-cpu-bisect.md | 4 +++- bench/README.md | 6 ++++-- rs/moq-net/src/model/frame.rs | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md index 676c7ca8a0c..30fb7c6bdbf 100644 --- a/.github/relay-cpu-bisect.md +++ b/.github/relay-cpu-bisect.md @@ -182,7 +182,7 @@ What remains of each viewer's live read is accepted in the [release-perf epic](. ## `group_relay_frames` since `e7a5c10` -[Fastest-reader pacing](https://github.com/moq-dev/moq/pull/7664)'s comparison found `group_relay_frames` about 15% slower at [f37d964dd3](https://github.com/moq-dev/moq/commit/f37d964dd3) than at [e7a5c10913](https://github.com/moq-dev/moq/commit/e7a5c10913) on a loaded host. The `group` fan-out scenario counts the same drain (every subscriber reads a finished 512-frame group in 16-frame batches), per drained group (runs yLeXzU, S9A3hY, R849HM): +[Fastest-reader pacing](https://github.com/moq-dev/moq/pull/7664)'s comparison found `group_relay_frames` about 15% slower at [f37d964dd3](https://github.com/moq-dev/moq/commit/f37d964dd3) than at [e7a5c10913](https://github.com/moq-dev/moq/commit/e7a5c10913) on a loaded host. The original `group` fan-out scenario counted the same drain (every subscriber reads a finished 512-frame group in 16-frame batches), per drained group (runs yLeXzU, S9A3hY, R849HM): | Revision | Instructions | Change | | --- | ---: | ---: | @@ -198,6 +198,8 @@ What remains of each viewer's live read is accepted in the [release-perf epic](. The +15.3% matches the wall-clock report. Nine other candidates between pacing and `f37d964dd3` count the same as their predecessor. [moq-dev/moq#7504](https://github.com/moq-dev/moq/pull/7504) put an idle gate in front of every frame read and split the read into a second out-of-line layer called from two branches, so each frame paid a call frame and the gate's bookkeeping, about 38 instructions on top of about 297. A frame already in the reader's prefetched batch now returns before the gate, the drift budget, or a route, none of which can apply to it (a read parks only with its batch drained, and a copy adopted mid-group starts empty); what remains below has one call site. The drain is now 9.4% below `e7a5c10913`, pacing included. +The counts above used the whole-frame batch reader before the maintainer made it private. `group_relay_frames` and the `group` scenario now use the relay's `poll_read_objects` through the hidden `fuzz::ObjectBatch` helper, retaining object metadata. These historical counts do not measure that revised harness. Pacing's isolated 2.0% cost was accepted and its dedicated quest removed. + ## Validation Every count, delivery estimate, loss figure, and host load cited here is in [the evidence](relay-cpu-bisect.json), keyed by run artifact. Raw captures (`perf.data`, folded stacks, reports) stay in the measuring checkout's ignored `.tmp/bench/`; the per-function figures for public ingest are recorded with run YxLYgw. diff --git a/bench/README.md b/bench/README.md index 37b5cb93bfb..ab97f72e357 100644 --- a/bench/README.md +++ b/bench/README.md @@ -437,8 +437,10 @@ run by default: QUIC: in-tree qmux, receive leases, and crypto only show in relay counts. A revision whose session API predates this harness reports it unsupported. - `group`, on request: every subscriber drains one finished 512-frame group in - object batches through the relay's private reader, as moq-net's `group_relay_frames/plain/512/*` bench does, so that - bench's cost is counted without its wall-clock noise. + object batches through the relay's private reader, as moq-net's + `group_relay_frames/plain/512/*` bench does, so that bench's cost is counted + without its wall-clock noise. Revisions before the hidden `ObjectBatch` helper + report unsupported; historical frame-batch counts remain in the bisect report. Each `bench/fanout/` crate builds against each base's and the current tree's own crates and lock, `BENCH_BUILD_JOBS` at a time, into diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index 2d3616a15e7..45cf1d036f9 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -12,6 +12,7 @@ use std::sync::{Arc, Weak}; use std::sync::{Mutex, OnceLock, PoisonError}; use std::task::{Poll, ready}; +#[cfg(any(test, feature = "fuzz"))] use arrayvec::ArrayVec; use bytes::{Buf, Bytes, BytesMut}; @@ -342,6 +343,7 @@ impl Buffer { } /// Whether the buffer holds no frames. + #[cfg(test)] pub fn is_empty(&self) -> bool { self.0.is_empty() } From 531c52d90a72a73c38d88d63c4315bfa36143f65 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 09:05:17 -0800 Subject: [PATCH 12/16] fix(net): retain private batch buffer emptiness check --- rs/moq-net/src/model/frame.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index 45cf1d036f9..324bcc0702d 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -343,7 +343,6 @@ impl Buffer { } /// Whether the buffer holds no frames. - #[cfg(test)] pub fn is_empty(&self) -> bool { self.0.is_empty() } From b253b76364f902f8ec5877173c630fd500b767d1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 09:11:34 -0800 Subject: [PATCH 13/16] docs(bench): record relay object batch baseline --- .github/relay-cpu-bisect.json | 35 +++++++++++++++++++++++++++++++++-- .github/relay-cpu-bisect.md | 2 ++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/.github/relay-cpu-bisect.json b/.github/relay-cpu-bisect.json index 0ac8ee20b57..3e741d0e0d0 100644 --- a/.github/relay-cpu-bisect.json +++ b/.github/relay-cpu-bisect.json @@ -7991,5 +7991,36 @@ ], "notes": [ "nixos2:run.cAVCAV (in-tree qmux and receive leases against their parents) was stopped after one round when other agents started fuzzing and compiling there; its round-1 cells (qmux 116k to 122k, leases 130k to 132k) are not conclusive and are not used." - ] -} + ], + "relay_object_batch": { + "artifact": "run.Qn1sqT", + "command": "just bench-fanout '' group '1 8 64'", + "metric": "user instructions per delivered 512-frame group, relay poll_read_objects, capacity 16", + "cells": [ + { + "label": "current", + "revision": "531c52d90a72a73c38d88d63c4315bfa36143f65", + "scenario": "group", + "subscribers": 1, + "cycles": 2000, + "per_group": 144907.2155 + }, + { + "label": "current", + "revision": "531c52d90a72a73c38d88d63c4315bfa36143f65", + "scenario": "group", + "subscribers": 8, + "cycles": 250, + "per_group": 144901.0835 + }, + { + "label": "current", + "revision": "531c52d90a72a73c38d88d63c4315bfa36143f65", + "scenario": "group", + "subscribers": 64, + "cycles": 32, + "per_group": 144900.27734375 + } + ] + } +} \ No newline at end of file diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md index 30fb7c6bdbf..b92c43138cb 100644 --- a/.github/relay-cpu-bisect.md +++ b/.github/relay-cpu-bisect.md @@ -200,6 +200,8 @@ The +15.3% matches the wall-clock report. Nine other candidates between pacing a The counts above used the whole-frame batch reader before the maintainer made it private. `group_relay_frames` and the `group` scenario now use the relay's `poll_read_objects` through the hidden `fuzz::ObjectBatch` helper, retaining object metadata. These historical counts do not measure that revised harness. Pacing's isolated 2.0% cost was accepted and its dedicated quest removed. +The revised object-batch harness at [531c52d90a](https://github.com/moq-dev/moq/commit/531c52d90a72a73c38d88d63c4315bfa36143f65) measures 144,907, 144,901, and 144,900 instructions per drained group at 1, 8, and 64 subscribers (`just bench-fanout "" group "1 8 64"`, run Qn1sqT). This is the new harness's baseline, with no historical comparison. + ## Validation Every count, delivery estimate, loss figure, and host load cited here is in [the evidence](relay-cpu-bisect.json), keyed by run artifact. Raw captures (`perf.data`, folded stacks, reports) stay in the measuring checkout's ignored `.tmp/bench/`; the per-function figures for public ingest are recorded with run YxLYgw. From 32cd099908e1135fde605a7258f468f582a0dc67 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 09:11:46 -0800 Subject: [PATCH 14/16] style(bench): terminate evidence JSON with newline --- .github/relay-cpu-bisect.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/relay-cpu-bisect.json b/.github/relay-cpu-bisect.json index 3e741d0e0d0..4614c15cfab 100644 --- a/.github/relay-cpu-bisect.json +++ b/.github/relay-cpu-bisect.json @@ -8023,4 +8023,4 @@ } ] } -} \ No newline at end of file +} From 92300424fba18645f87674b37386402e2f0b17ff Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 09:16:52 -0800 Subject: [PATCH 15/16] fix(bench): distinguish relay object batch benchmark IDs --- .github/relay-cpu-bisect.md | 2 +- bench/README.md | 2 +- bench/fanout/group/src/main.rs | 2 +- rs/moq-net/benches/group.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md index b92c43138cb..01116a8268b 100644 --- a/.github/relay-cpu-bisect.md +++ b/.github/relay-cpu-bisect.md @@ -198,7 +198,7 @@ What remains of each viewer's live read is accepted in the [release-perf epic](. The +15.3% matches the wall-clock report. Nine other candidates between pacing and `f37d964dd3` count the same as their predecessor. [moq-dev/moq#7504](https://github.com/moq-dev/moq/pull/7504) put an idle gate in front of every frame read and split the read into a second out-of-line layer called from two branches, so each frame paid a call frame and the gate's bookkeeping, about 38 instructions on top of about 297. A frame already in the reader's prefetched batch now returns before the gate, the drift budget, or a route, none of which can apply to it (a read parks only with its batch drained, and a copy adopted mid-group starts empty); what remains below has one call site. The drain is now 9.4% below `e7a5c10913`, pacing included. -The counts above used the whole-frame batch reader before the maintainer made it private. `group_relay_frames` and the `group` scenario now use the relay's `poll_read_objects` through the hidden `fuzz::ObjectBatch` helper, retaining object metadata. These historical counts do not measure that revised harness. Pacing's isolated 2.0% cost was accepted and its dedicated quest removed. +The counts above used the whole-frame batch reader before the maintainer made it private. `group_relay_objects` (renamed to prevent comparisons with the old frame-batch IDs) and the `group` scenario now use the relay's `poll_read_objects` through the hidden `fuzz::ObjectBatch` helper, retaining object metadata. These historical counts do not measure that revised harness. Pacing's isolated 2.0% cost was accepted and its dedicated quest removed. The revised object-batch harness at [531c52d90a](https://github.com/moq-dev/moq/commit/531c52d90a72a73c38d88d63c4315bfa36143f65) measures 144,907, 144,901, and 144,900 instructions per drained group at 1, 8, and 64 subscribers (`just bench-fanout "" group "1 8 64"`, run Qn1sqT). This is the new harness's baseline, with no historical comparison. diff --git a/bench/README.md b/bench/README.md index ab97f72e357..a9a9672cf3d 100644 --- a/bench/README.md +++ b/bench/README.md @@ -438,7 +438,7 @@ run by default: revision whose session API predates this harness reports it unsupported. - `group`, on request: every subscriber drains one finished 512-frame group in object batches through the relay's private reader, as moq-net's - `group_relay_frames/plain/512/*` bench does, so that bench's cost is counted + `group_relay_objects/plain/512/*` bench does, so that bench's cost is counted without its wall-clock noise. Revisions before the hidden `ObjectBatch` helper report unsupported; historical frame-batch counts remain in the bisect report. diff --git a/bench/fanout/group/src/main.rs b/bench/fanout/group/src/main.rs index e5c24ff73f8..6e7283d3e6c 100644 --- a/bench/fanout/group/src/main.rs +++ b/bench/fanout/group/src/main.rs @@ -1,5 +1,5 @@ //! A relay's frame fan-out: every subscriber drains one finished 512-frame group in batches, -//! as moq-net's `group_relay_frames/plain/512/*` bench does. +//! as moq-net's `group_relay_objects/plain/512/*` bench does. //! Usage: `fanout-group SUBSCRIBERS CYCLES`. The work is deterministic, so its retired //! instructions, less a run of zero cycles, are the per-group cost whatever the host does. use std::task::Poll; diff --git a/rs/moq-net/benches/group.rs b/rs/moq-net/benches/group.rs index 3473a804741..df894e437ef 100644 --- a/rs/moq-net/benches/group.rs +++ b/rs/moq-net/benches/group.rs @@ -362,7 +362,7 @@ fn bench_roundtrip(c: &mut Criterion) { fn bench_relay_frames(c: &mut Criterion) { #[cfg(feature = "fuzz")] { - let mut g = c.benchmark_group("group_relay_frames"); + let mut g = c.benchmark_group("group_relay_objects"); for frames in [4, 512, 8192] { for subscribers in [1, 8, 64] { for properties in [false, true] { From 8a13e4dd7774278bcffbfe4f53a40d554c6cd102 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 10:11:53 -0800 Subject: [PATCH 16/16] fix(net): a drop poll still lets go of a replaced copy The empty drop report returned before let_go, so a replaced copy kept only by a group handed out from it stayed subscribed after that group was dropped: nothing else wakes the parked group read. The early return now applies only while no copy is replaced, the single-route hot path. Also corrects the bisect's run agreement claim and the group scenario's method note. Co-Authored-By: Claude Opus 5.5 --- .github/relay-cpu-bisect.json | 2 +- .github/relay-cpu-bisect.md | 2 +- rs/moq-net/src/model/resume.rs | 32 +++++++++++++++++++++++++++++--- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/.github/relay-cpu-bisect.json b/.github/relay-cpu-bisect.json index 4614c15cfab..3af44c2f882 100644 --- a/.github/relay-cpu-bisect.json +++ b/.github/relay-cpu-bisect.json @@ -3480,7 +3480,7 @@ ] } ], - "fanout_method": "just bench-fanout: user instructions per delivered group on one pinned thread, a run of many cycles less a run of zero, at the listed subscriber counts; model = the track model alone, session = publisher, relay origin, and viewers over the in-memory transport; group = every subscriber drains one finished 512-frame group in 16-frame batches, as group_relay_frames/plain/512 does, per drained group", + "fanout_method": "just bench-fanout: user instructions per delivered group on one pinned thread, a run of many cycles less a run of zero, at the listed subscriber counts; model = the track model alone, session = publisher, relay origin, and viewers over the in-memory transport; group = every subscriber drains one finished 512-frame group in 16-frame batches, per drained group; the recorded group runs read whole frames as group_relay_frames/plain/512 did, while the scenario now reads objects through fuzz::ObjectBatch::<16> as group_relay_objects/plain/512 does, so its new counts do not compare with them", "fanout_runs": [ { "artifact": "hist", diff --git a/.github/relay-cpu-bisect.md b/.github/relay-cpu-bisect.md index 01116a8268b..53fc41b88b3 100644 --- a/.github/relay-cpu-bisect.md +++ b/.github/relay-cpu-bisect.md @@ -49,7 +49,7 @@ Re-paired on the quiet test host `nixos2` (an i7-8700: relay on CPUs 0-3, genera | video, io\_uring | TZ2ODR | 99.2 \[97.4-102.2] | 110.8 \[108.6-112.9] | 1.12 | 1.1 | | video, tokio workers | 6xoc9u | 136.5 \[128.7-139.8] | 121.6 \[118.6-124.1] | 0.89 | 1.6 | -Medians of the quiet cells in four rounds, min-max in brackets: every cell used delivered its workload's rate within 2% with no loss, at a one-minute load of at most 6. Two cells were dropped: the old endpoint's fourth chat round on io\_uring (load 8.5) and its second on the tokio workers (load 8.7, 3.9% short of the rate). Run 4U16wN's `main` and run ssZrgC's (142.3 \[141.6-143.1], three rounds) agree within 0.3k with run S0G1Z7's. +Medians of the quiet cells in four rounds, min-max in brackets: every cell used delivered its workload's rate within 2% with no loss, at a one-minute load of at most 6. Two cells were dropped: the old endpoint's fourth chat round on io\_uring (load 8.5) and its second on the tokio workers (load 8.7, 3.9% short of the rate). Run ssZrgC's `main` (142.3 \[141.6-143.1], three rounds) agrees within 0.3k with run S0G1Z7's. The earlier pairing on the 24-core host, at load up to 40 for the video rows, read chat 1.26 on io\_uring and 1.31 on the tokio workers, and video 1.14 and 0.96. Only chat on the tokio workers moved by more than those runs' spread, and its earlier run reached load 23 on another host, so the change is not attributed. Chat on io\_uring remains the workload furthest above the old endpoint. By the nearest named function, chat's growth on `main` at [c68c931025](https://github.com/moq-dev/moq/commit/c68c931025), before the fast path, falls in the same places on io\_uring (run.qYPSii) and the tokio workers (run.IV47ua), in instructions per frame: diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 3ef40218e6b..af8b259d25d 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -1763,9 +1763,10 @@ impl Reader { for run in covered { self.drops.remove(run); } - // The common case: nothing to report, and the serving copy's own poll above wakes - // this for the next run it gives up, so no copy needs judging. - if self.drops.is_empty() { + // The common case: nothing to report on one route, and the serving copy's own poll + // above wakes this for the next run it gives up. A replaced copy is still judged: + // dropping the last group handed out from it wakes nothing else that lets it go. + if self.drops.is_empty() && !self.copies.iter().any(|copy| copy.replaced) { return Poll::Pending; } // Judged as the group path judges it, so a copy kept only for a group still read @@ -4936,6 +4937,31 @@ mod test { } } + /// A drop poll with nothing to report still lets go of a replaced copy once the last + /// group handed out from it is dropped: nothing wakes the parked group read for that. + #[test] + #[cfg(feature = "async")] + fn a_drop_poll_lets_go_a_replaced_copy_once_its_group_is_dropped() { + let routes = Producer::new(); + let logical = logical(&routes); + let a = copy(); + routes.serve(a.consume()); + let mut sub = subscribe(&logical, HOUR); + write(&a, 0); + let Poll::Ready(Ok(Some(held))) = sub.poll_recv_group(&kio::Waiter::noop()) else { + panic!("expected group 0"); + }; + let b = copy(); + routes.serve(b.consume()); + write(&b, 1); + assert_eq!(drain(&mut sub, false), [1]); + assert_eq!(copies(&routes), 2, "the held group keeps its copy"); + + drop(held); + assert!(sub.poll_drop(&kio::Waiter::noop()).is_pending()); + assert_eq!(copies(&routes), 1, "the replaced copy was let go"); + } + /// A group the new copy replays, already handed out, does not bound the replaced copy: /// the old route's next group still arrives. #[test]