From 4387688a38f9cf99e29f020a87e9057cdbd7a258 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 04:21:06 -0700 Subject: [PATCH 01/13] chore(quest): claim quest/m1/release-perf/README.md Quest-Claim: b8518661d44554fd From e03bf01e9a4fc9c8d97cb0774c51d3a61bda6921 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 05:55:07 -0700 Subject: [PATCH 02/13] fix(bench): skip targets without Criterion output when pairing a base `just bench ` discovers each target's cases from a short Criterion run, and failed outright on a harness = false target that prints its own table, such as moq-cli's play_buffer. Such a target is now listed as skipped, so a base whose bench later became Criterion reports its cases as new. Co-Authored-By: Claude Opus 5.5 --- bench/run.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bench/run.sh b/bench/run.sh index 89aedccb864..78fa26535aa 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -435,6 +435,11 @@ criterion_cases() { CRITERION_ARGS=(--warm-up-time 0.001 --measurement-time 0.001 --sample-size 10 --nresamples 1 --noplot) CRITERION_HOME=$discovery run_criterion_target "$checkout" "$manifest" "$target" --save-baseline discovery ) >&2 || return 1 + # A harness = false target without Criterion, such as moq-cli's play_buffer, writes no samples. + if [[ -z $(find "$discovery" -name benchmark.json -print -quit 2>/dev/null) ]]; then + printf 'Not a Criterion target, skipped: %s %s\n' "$manifest" "$target" >&2 + continue + fi python3 "$ROOT/bench/compare.py" --criterion-cases "$discovery" >"$RUN/criterion-listing" || return 1 while IFS= read -r benchmark; do printf '%s\t%s\t%s\n' "$manifest" "$target" "$benchmark" From fb89e97e2a045e2a46b28eda8f8ebbff98b1fc59 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 05:55:13 -0700 Subject: [PATCH 03/13] fix(mux): keep catalog_write_order within the rendition cap Its 256-track case panicked with TooManyRenditions, since one catalog holds at most hang::catalog::MAX_RENDITIONS (64). The sweep now tops out there. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/benches/catalog.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rs/moq-mux/benches/catalog.rs b/rs/moq-mux/benches/catalog.rs index 299a650ac13..b7e532e875e 100644 --- a/rs/moq-mux/benches/catalog.rs +++ b/rs/moq-mux/benches/catalog.rs @@ -222,8 +222,8 @@ fn bench_unresolved_teardown(c: &mut Criterion) { group.finish(); } -/// Tracks interleaving frames on one catalog. -const ORDERED: [usize; 3] = [1, 16, 256]; +/// Tracks interleaving frames on one catalog, up to the most renditions it can hold. +const ORDERED: [usize; 3] = [1, 16, hang::catalog::MAX_RENDITIONS]; /// Frames each track writes per iteration. const FRAMES: u64 = 100; From 5ad3ef9fc6074190c42ef147528889dad4f741fd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 05:55:20 -0700 Subject: [PATCH 04/13] fix(relay): stop the auth bench waiting for a grant it already has Since #6109, Handle::add resolves once the peer grants the token, so a watch created afterwards already holds that grant and changed() waited forever. relay_auth_refresh hung before its first case. Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/benches/auth.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rs/moq-relay/benches/auth.rs b/rs/moq-relay/benches/auth.rs index d0a404be3ae..495acc71b34 100644 --- a/rs/moq-relay/benches/auth.rs +++ b/rs/moq-relay/benches/auth.rs @@ -84,8 +84,9 @@ impl Fixture { let added = futures::future::join_all((0..tokens).map(|index| auth.add(format!("token{index}")))).await; let added: Vec<_> = added.into_iter().map(|added| added.unwrap().0).collect(); if session == 0 { - let mut observed = added.last().unwrap().grant(); - observed.changed().await.unwrap(); + // `add` resolves once the peer grants the token, so its grant is already current. + let observed = added.last().unwrap().grant(); + assert!(observed.peek().is_some()); watch = Some(observed); } for _ in 0..tokens { From e830aec82f52aea133005046affb61945a56e417 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 05:55:26 -0700 Subject: [PATCH 05/13] fix(stats): stop the four-tier tick sweep below the snapshot cap Since #6428 every tier shares the one depth-0 group's snapshot, so 50,000 paths over four tiers (about 86 MB) outgrow the 32 MB frame cap and the track ends with FRAME_TOO_LARGE, as doc/concept/stats.md describes. The case measured an ended track and failed its own check; one tier still sweeps to 50,000. Co-Authored-By: Claude Opus 5.5 --- rs/moq-stats/benches/producer_tick.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rs/moq-stats/benches/producer_tick.rs b/rs/moq-stats/benches/producer_tick.rs index 1c13e68389d..75a35c5ec8d 100644 --- a/rs/moq-stats/benches/producer_tick.rs +++ b/rs/moq-stats/benches/producer_tick.rs @@ -76,7 +76,8 @@ unsafe impl GlobalAlloc for Counter { /// Ticks averaged into one table row, after a warm-up drain. const TABLE_TICKS: u32 = 4; -/// Held paths. The last count is sized so its plain snapshot nears the cache cap; `cap%` shows how close. +/// Held paths. The last count is sized so its one-tier snapshot nears the cache cap; `cap%` shows +/// how close. Every tier shares the one group's snapshot, so more tiers stop below it. const PATHS: [usize; 4] = [100, 1_000, 10_000, 50_000]; const TIERS: [usize; 2] = [1, 4]; @@ -235,6 +236,9 @@ fn bench(c: &mut Criterion) { for &changed in &CHANGED { for &tiers in &TIERS { for &paths in &PATHS { + if tiers > 1 && paths == PATHS[PATHS.len() - 1] { + continue; + } let id = BenchmarkId::from_parameter(format!("{paths}p_{tiers}t_{changed}pct")); group.throughput(Throughput::Elements((paths * tiers) as u64)); // Criterion re-enters the closure per sample, so build and print the row From 35d4265ce3d1289e5c01ad090304d6ab6a2073e7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 05:55:56 -0700 Subject: [PATCH 06/13] docs(bench): note that pairing skips targets without Criterion samples Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/README.md b/bench/README.md index 61a13828c53..eb317761c3d 100644 --- a/bench/README.md +++ b/bench/README.md @@ -219,7 +219,8 @@ source files. The retained path is printed at the end, including on failure. Choose an evidence parent outside disposable worktrees when the report must survive worktree removal. Criterion case discovery uses isolated short measurements and their full JSON IDs, since `--list` truncates display names; discovery samples -are retained separately and never used in the comparison. +are retained separately and never used in the comparison. A target that writes +no Criterion samples, such as one that prints its own table, is listed as skipped. Each comparison reports the median of the paired percentage deltas, their median absolute deviation and range, and flags fewer than three observations or a sign From b270b366a22ccf10670c3b2d3ac7fc55643d5d13 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 08:14:47 -0700 Subject: [PATCH 07/13] docs(quest): add a release baseline step to the performance audit Co-Authored-By: Claude Opus 5.5 --- quest/m2/audit-perf.md | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/quest/m2/audit-perf.md b/quest/m2/audit-perf.md index cb5a9649491..bebfbdc39f1 100644 --- a/quest/m2/audit-perf.md +++ b/quest/m2/audit-perf.md @@ -4,15 +4,18 @@ Every benchmark that regressed since the previous run is explained, fixed, or planned, and every hot or fan-out path without a benchmark gets one or a -follow-up. +follow-up. `main` ships no slower than `release`: every case where the cut +this run gates would be slower is fixed or accepted with the feature it pays +for. ## Plan Recurs once per major release (decided 2026-10-19). The PR that finishes this run also adds the next one with the same file name at the end of the next -milestone's `Required` (m0 to m1, m1 to m2). The successor keeps the Goal -and this paragraph; the rest of its Plan is rewritten with what this run -learned: baselines, results, and what to check first. A successor placed in m2 +milestone's `Required` (m0 to m1, m1 to m2). The successor keeps the Goal, +this paragraph, and the release baseline section; the rest of its Plan is +rewritten with what this run learned: baselines, results, and what to check +first. A successor placed in m2 also lists the [release cut](/quest/m2/release/cut.md) under `Required`, so it waits for the release in flight to ship. Only `/quest-delete` ends the recurrence. Fixes land in the run's PR, one commit each, rather than as follow-ups. @@ -20,6 +23,26 @@ Only a change that needs a maintainer decision (public API, scope) or is independently landable and too large to review alongside the rest becomes a follow-up through `/quest-plan`. +### Release baseline + +The run's last step, once the rest of its milestone has merged, so `main` is +what the cut ships. Pair it with the last cut, keeping the evidence outside +the checkout, and record `/proc/loadavg` around each run: + +```bash +BENCH_ARTIFACTS= just bench origin/release +bun bench/browser/micro.ts --bench frames --out /frames.json base= current=. aa=. +``` + +`` is an export of `origin/release` with its own +`bun install --frozen-lockfile`. Re-run every slower case in alternating +rounds and count instructions for any that holds (see Method). A ratio +against `release` shows where to look, not what to fix: `main` carries +features `release` lacks, so a flat per-operation cost usually pays for one, +while a cost that grows with `main`'s own sweep is accidental. Fix accidental +cost in this run, and record each accepted cost in its results with the +feature and PR it pays for. A browser row inside the `aa` spread is noise. + ### What the m1 run (2026-10-22) fixed Instruction counts are one optimized `--test --exact` iteration under callgrind, From c826d82f65a85708ac04b39e145938363bc33b68 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 09:14:51 -0700 Subject: [PATCH 08/13] chore(quest): finish the main against release comparison Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 - quest/m1/api-audit/README.md | 4 - quest/m1/release-audit/README.md | 1 - quest/m1/release-perf/README.md | 168 -------------------------- quest/m2/performance-survey/README.md | 7 +- 5 files changed, 4 insertions(+), 177 deletions(-) delete mode 100644 quest/m1/release-perf/README.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 84cf6e6c7b1..9fa5a3705e3 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -145,7 +145,6 @@ Catalog HLS field. - [WebGPU on Safari](/quest/m1/webgpu-safari.md) - the WebGPU renderer is verified on Safari 26 for macOS and iOS - [JSON encoder options without a downcast](/quest/m1/json-encoder-options.md) - moq-mux's `json_snapshot` takes its `deltas` switch through a typed path any custom catalog entry can use, not an `Any` downcast - [CI housekeeping](/quest/m1/ci-housekeeping.md) - validate queue nextest at 24 threads and remove the actionlint queue ignore after its release -- [Main is no slower than release](/quest/m1/release-perf/README.md) - every benchmark where main is slower than release is fixed or accepted with a reason, checked by a final paired run before the cut - [Drop announce="never"](/quest/m1/publish-announce-never.md) - `@moq/publish` offers only announce modes a watcher can reach - [Drop unused OBS outputs](/quest/m1/obs-output-variants.md) - the plugin registers only `moq_output`, so the broken audio-only variant is gone - [API Extractor checks](/quest/m1/api-extractor-checks.md) - API Extractor fails a PR on a forgotten export, an unresolved declaration, or an undocumented export, replacing the hand-rolled `.d.ts` checkers diff --git a/quest/m1/api-audit/README.md b/quest/m1/api-audit/README.md index f382640a3e4..533e165af92 100644 --- a/quest/m1/api-audit/README.md +++ b/quest/m1/api-audit/README.md @@ -74,7 +74,3 @@ Rejected in planning: - [One lifecycle vocabulary](/quest/m1/api-audit/lifecycle.md) - finish, abort, closed, and close mean one thing on every handle, and Rust and JS observers fire alike - [moq-net names say one thing](/quest/m1/api-audit/net-names.md) - demand says `wanted`, colliding stats and transport names are renamed, and Rust and JS info lookups mirror - [Request handlers are named requests](/quest/m1/api-audit/requests.md) - `dynamic()` and `Dynamic` become `requests()` and `Requests` at every layer - -## Related - -- [Main is no slower than release](/quest/m1/release-perf/README.md) - the other release-readiness check diff --git a/quest/m1/release-audit/README.md b/quest/m1/release-audit/README.md index fbd79026a82..4dd167cc32e 100644 --- a/quest/m1/release-audit/README.md +++ b/quest/m1/release-audit/README.md @@ -98,7 +98,6 @@ child records whether its behavior is new on main or inherited from release. - [JS subscription ranges](/quest/m1/subscribe-ranges/js.md) - range lists and frame offsets build on the bounds fixed here - [OBS video source](/quest/m2/obs-moq-video/source.md) - the decoder replacement keeps the timing and EOF behavior fixed here - [Compressed tracks](/quest/m1/compress/README.md) - application migration has an existing owner -- [Measured release performance](/quest/m1/release-perf/README.md) - existing paired benchmark regressions and their confirmation runs - [Transport performance audit](/quest/m2/audit-perf.md) - congestion-control coverage remains outstanding - [Binding timestamps](/quest/m1/binding-timestamps.md) - exact timestamps across language APIs - [Dependency audit](/quest/m2/audit-deps.md) - existing advisory inventory diff --git a/quest/m1/release-perf/README.md b/quest/m1/release-perf/README.md deleted file mode 100644 index c9619d210fc..00000000000 --- a/quest/m1/release-perf/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# [M] Main is no slower than release - -## Goal - -Before the next release cut, every benchmark where `main` is slower than -`release` is either fixed or accepted with a recorded reason. Accidental cost -is fixed. Cost that pays for a feature `main` deliberately added is accepted, -and the reason is written here. - -This epic's own work: once every child has landed, run the full main against -release comparison again with `just bench origin/release`, plus the JS frame -path in Chromium with `bench/browser/micro.ts`, and confirm nothing new -regressed. Then add a release-baseline step to the recurring -[performance audit](/quest/m2/audit-perf.md), so every later cut -repeats this comparison. - -## Plan - -Decided 2026-10-21, when planning from a paired comparison of `release` -(`0c56e5b4e`, which included the bench backport -[#6103](https://github.com/moq-dev/moq/pull/6103)) against `main` -(`8b0d9952f`). The run used 811 paired Criterion cases, three alternating -confirmation rounds for the largest moves, JS on bun, and the relay load -harness. It ran on a shared 8-core host, so a small wall-time change may be -noise. - -- The epic gates the cut through the [m1 milestone](/quest/m1/README.md), - because `main` must not ship slower than `release` without a decision. -- In scope: the five cases that held through confirmation, plus the large - screen-only moves. The screen-only moves are re-measured with callgrind - instruction counts, and any that hold are fixed. -- Only accidental cost is fixed. Cost from an intended feature is accepted, - and the reason is recorded here. -- The IETF `/fetch` rows of `session_delivery_group_cost` are accepted and - dropped from release pairing. They measure different work on each branch: - release's bare track handle opened a live SUBSCRIBE, while main sends a real - FETCH per viewer ([#5820](https://github.com/moq-dev/moq/pull/5820) made - handles lazy, and [#4974](https://github.com/moq-dev/moq/pull/4974) added - fetch-only demand). The lite `/fetch` cost was real: a relay sent one - upstream FETCH per group, but revived the parked logical track for every - viewer's FETCH. [#6286](https://github.com/moq-dev/moq/pull/6286) serves a - whole cached group from the parked copy instead, cutting 12 to 19% of the - instructions per viewer where viewers share a group. The rest of the gap to - a viewer that holds a query is accepted: each viewer's own fetch of a group - it lacks revives its logical track, so the open group keeps its route and - fails over mid-read, and its copy releases the route's demand once idle. -- `stats/session-report` is accepted at O(live sessions), after a cheap fix. - Per-label payload bytes ([#5731](https://github.com/moq-dev/moq/pull/5731)) - match payload billing per label, such as per signing key. After removing - redundant row locking and live-session CAS, three alternating measurements - give 52.2 us for 10,000 sessions without wires and 388.4 us with one real - transport wire each. A once-per-second drain uses about 0.04% of a core - with those wires; the earlier under-0.01% estimate omitted their walk. - This remaining per-session and per-connection sampling is accepted. Deleting - per-label bytes was rejected. -- `track_gc/idle` is accepted at about 20 ns, flat across track counts, after - [#6291](https://github.com/moq-dev/moq/pull/6291) skipped the clock read - with no wall deadline and reused the bench's waiter (36.7 ns before). The - remaining ~8 ns over release's 11.6 ns is the lock and driver registration - that the wall-clock aging from - [#5548](https://github.com/moq-dev/moq/pull/5548) needs. An earliest-deadline - atomic cannot prove a replacement driver is registered, and a lock-free - notification primitive is not worth ~8 ns. -- `track_gap_flush` is accepted at O(log gaps + retired entries) per shared - successor, after [#6287](https://github.com/moq-dev/moq/pull/6287) indexed - the gap ledger and retired a shared verdict's gaps with one split. Each - retired entry must be freed exactly once, so the batch is already amortized - optimal; a representation that hides that reclamation would widen scope for - no measured benefit. Sparse arrivals cost O(log gaps) amortized, and at 4096 - gaps flush, read, and expiry instructions fell 80 to 95%. -- The mux exports' and Annex B import's fixed cost per broadcast is accepted. - It publishes required output: a per-track timeline and an immediately - completed catalog record. Their cold DEFLATE state is most of it (81% of a - one-frame import's instructions are its zero-fill), and the encoders already - build that state lazily on first use. It is paid once per broadcast: the - import ratio falls from 3.97x at one frame to 1.49x at 64. TS export's - growth to 2.09x at `4x64` is in the export reap that - [moq-dev/moq#6281](https://github.com/moq-dev/moq/pull/6281) changes. -- The model's remaining per-group and per-frame cost is accepted, after - [moq-dev/moq#6294](https://github.com/moq-dev/moq/pull/6294) cut the - accidental part (a second track upgrade per first frame, a 128-bit division - per buffered datagram on join, and draining empty waiter lists on every - write). What remains pays for features: the arrival stamp each frame reads - from the clock ([#6249](https://github.com/moq-dev/moq/pull/6249)), largest - notifications ([#5725](https://github.com/moq-dev/moq/pull/5725)), the - rewind check and starts index each first frame fills - ([#5803](https://github.com/moq-dev/moq/pull/5803), - [#6074](https://github.com/moq-dev/moq/pull/6074)), indexed expiry wakes, - the bounded wall-clock expiry scan per append, and a subscriber's ranges and - datagram max delay ([#5195](https://github.com/moq-dev/moq/pull/5195)). - None of it grows with the sweeps. `track_stamp_groups/sparse` stays highest - at 4.4x, because release's first frame skipped nearly all of that work. -- Session delivery is accepted at 1.28x to 1.77x in instructions. It is the - per-group and per-frame cost above on every hop, measured through the - flow-controlled mock transport - ([#5530](https://github.com/moq-dev/moq/pull/5530)) that release's harness - lacked. - -### Release as a baseline - -`release` lacks many of the features the backported benches exercise: -per-reader delivery records, wall-clock aging, the gap ledger, rewind checks, -and epochs. So a ratio against `release` shows where to look, not what to fix. -Main's own sweeps over a size parameter are the better regression signal: a -cost that grows with the sweep is accidental, while a flat per-operation cost -usually pays for a feature. - -### Confirmed (three rounds, ratio is main / release) - -| Bench | release | main | Ratio | Owner | -|---|---|---|---|---| -| `track_gap_flush/4096` | 5.7 us | 551 us | 96x | accepted (above) | -| `track_gap_flush/512` | 1.8 us | 48 us | 27x | accepted (above) | -| `track_gap_flush/64` | 1.5 us | 6.2 us | 4.1x | accepted (above) | -| `stats/session-report/1000x10` | 11.0 us | 86.8 us | 7.9x | accepted (above) | -| `track_gc/idle/16384` | 11.6 ns | 73.7 ns | 6.3x | accepted (above) | -| `origin/resume_overlap/sequence/1r_64g_32c` | 93 us | 426 us | 4.6x | fixed in [moq-dev/moq#6292](https://github.com/moq-dev/moq/pull/6292) | -| `session_delivery_group_cost/moq-transport-22/.../viewers=64/.../fetch` | 1.19 ms | 3.86 ms | 3.2x | accepted (above) | -| `group_write_frames/single/8192` | 424 us | 486 us | 1.17x | accepted (above) | - -Main is much faster on subscriber churn, joins, range demand, and idle -sessions (by 100x to 3000x). The relay load test used 21% less CPU on video -and 12% less on fanout, with the same delivery. - -### Screen only, re-measured - -Re-measured on 2026-10-22 with callgrind instructions per operation, release -rebuilt under main's dev shell with fat LTO. Ratios are main over release; -the moq-net and kio rows include the fixes above. - -| Bench | Screen | Instructions | Owner | -|---|---|---|---| -| `container-export/ts/4x1`, `mkv/4x1` | 4.7x, 3.9x | 1.70x, 1.59x | accepted (above) | -| `annexb-import`, 1 and 64 frames | median 2.1x | 3.97x, 1.49x | accepted (above) | -| `track_stamp_groups/append/64`, `sparse/64` | median 2.7x | 2.21x, 4.43x | accepted (above) | -| `track_parallel_write/1` | 1.6x | 1.99x | accepted (above) | -| `track_fanout_frame/1` | 1.42x | 1.86x | accepted (above) | -| `track_empty_run/{fresh/1,stale/8}/512` | median 1.36x | 1.53x | accepted (above) | -| `track_datagram_subscriber_join/0d/1`, `64d/64` | median 2.3x | 2.12x, 3.55x | accepted (above) | -| `track_sparse_read/timed/4096` | 4.3x | | [moq-dev/moq#6287](https://github.com/moq-dev/moq/pull/6287) | -| `origin/announce_duplicate`, `1d_1s` to `240d_240s` | 4.3x | 1.06x to 1.16x | noise | -| `session_delivery_{relays,paced,scale}` | 1.43x to 1.56x | 1.28x to 1.77x | accepted (above) | -| moq-stats `aggregate`, live=16, departed=4096 | 2.1x | 1.05x | noise | -| kio `contended_{notify,log}`, 16 threads | 4x to 5x | wall 1.04x to 1.15x | noise | -| JS `net/bench/frames.ts` on bun | 1.2x to 2.2x | | accepted (below) | - -The kio contention benches spin across threads, so instructions don't measure -them; the row is the least loaded of three alternating wall-time rounds, and -later rounds swung both ways. - -JS `net/bench/frames.ts` on bun had lite rows at 1.2 to 1.3x and IETF rows at -1.3 to 2.2x. Fixed in Chromium: -[#6296](https://github.com/moq-dev/moq/pull/6296) removed a retained-history -scan, and [#6479](https://github.com/moq-dev/moq/pull/6479) stopped -releasing a finished stream's reader lock, which built a `TypeError` per -group stream. Every row then sits within the A/A spread of release. -The group write/read microbench stays 3 to 12% above release against an -A/A of 3 to 8% on the same host. Decided 2026-10-23 (maintainer): browser -parity accepted on this evidence; the final re-run above checks it again. - -The raw report lived in a scratch directory that has since been removed. -`just bench ` now pairs full Criterion IDs; point `BENCH_ARTIFACTS` -outside the worktree to keep the next one. - -## Related - -- [Performance audit](/quest/m2/audit-perf.md) - compares nightly to nightly; this epic compares main to release -- [Calibrate the paired nightly benchmarks](/quest/m1/bench-ci.md) - noise floor of the paired nightly-to-nightly comparison on one runner diff --git a/quest/m2/performance-survey/README.md b/quest/m2/performance-survey/README.md index ec9126c6d8a..95f5d0adbb9 100644 --- a/quest/m2/performance-survey/README.md +++ b/quest/m2/performance-survey/README.md @@ -143,9 +143,10 @@ No duplicate quest or workflow edit belongs here. ### Existing ownership and deliberately deferred work -- [Release performance](/quest/m1/release-perf/README.md) owns sparse gaps, - resume duplicates, idle GC, stats drain, FETCH fan-out, JS frame regressions, - and paired comparison correctness. Active PRs already address these. +- The [performance audit](/quest/m2/audit-perf.md) repeats the main against + release comparison before each cut. The m1 comparison already fixed or + accepted sparse gaps, resume duplicates, idle GC, stats drain, FETCH fan-out, + JS frame regressions, and paired comparison correctness. - [Perf](/quest/m2/perf/README.md) owns measured channel contention, relay allocations, ingest batching, worker quiescence and runtime selection. - The [release audit](/quest/m1/release-audit/README.md) bounded the media From 87b616dfaa8c529dcbd05a796c3319a69dcd3d48 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 10:42:58 -0700 Subject: [PATCH 09/13] perf(mux): set the MKV and FLV export clock's tracks once per catalog update Both exports rebuilt the jitter buffer's expected-track set on every poll, cloning and sorting every track name per frame in MKV. Tracks change only when the catalog does, so the set is now set there. container-export instructions: mkv/16x64 27.1M to 17.6M (-35%), flv/16x64 14.5M to 12.9M. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/container/flv/export.rs | 11 ++++++----- rs/moq-mux/src/container/mkv/export.rs | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/rs/moq-mux/src/container/flv/export.rs b/rs/moq-mux/src/container/flv/export.rs index f7334dea2c4..274155b919d 100644 --- a/rs/moq-mux/src/container/flv/export.rs +++ b/rs/moq-mux/src/container/flv/export.rs @@ -401,11 +401,6 @@ impl Export { } } - self.jitter.expect( - (0..self.video.len()) - .map(|i| (0, i)) - .chain((0..self.audio.len()).map(|i| (1, i))), - ); self.fill(waiter)?; // 3. Emit the header once every track's codec config has resolved. @@ -600,6 +595,12 @@ impl Export { // publisher retires a rendition as its track finishes or drops, and the catalog // update races that end on another stream, so only the track's end can say which it // was: a finish ends it cleanly, and a drop is the error the export reports. + // Only binding adds tracks, so the clock waits on the bound set as of this update. + self.jitter.expect( + (0..self.video.len()) + .map(|i| (0, i)) + .chain((0..self.audio.len()).map(|i| (1, i))), + ); Ok(()) } diff --git a/rs/moq-mux/src/container/mkv/export.rs b/rs/moq-mux/src/container/mkv/export.rs index 3df1957f5ab..a2ac5033026 100644 --- a/rs/moq-mux/src/container/mkv/export.rs +++ b/rs/moq-mux/src/container/mkv/export.rs @@ -391,7 +391,6 @@ impl Export { } } - self.jitter.expect(self.tracks.keys().cloned()); self.fill(waiter)?; // 3. Before the header is emitted: keep pulling until every video @@ -661,6 +660,8 @@ impl Export { next_track_number += 1; } + // Tracks change only before the header, so the clock waits on this set from here on. + self.jitter.expect(self.tracks.keys().cloned()); self.catalog_snapshot = Some(catalog); Ok(()) } From 51515e07f6b8c2304a1056b56c3c8734819a752d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 10:42:59 -0700 Subject: [PATCH 10/13] perf(mux): hash TS export PIDs with FxHash The T-STD schedule looks up per-PID buffers for every queued unit on each PCR slot, and SipHash was 18% of the export. PIDs are at most 13 bits from a bounded catalog, so a fast hasher is safe. container-export/ts/16x64 instructions 30.8M to 25.5M (-17%), ts/1x64 -12%. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + rs/moq-mux/Cargo.toml | 1 + rs/moq-mux/src/container/ts/export.rs | 5 +++-- rs/moq-mux/src/container/ts/schedule.rs | 29 ++++++++++++++----------- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3bb883965a7..5935f1dd93f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5022,6 +5022,7 @@ dependencies = [ "mpeg2ts", "num_enum", "roxmltree", + "rustc-hash 2.1.3", "scuffle-av1", "scuffle-h265", "serde", diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index 02e306534df..162d25cbb71 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -32,6 +32,7 @@ mp4-atom = { version = "0.16.2", features = ["tokio", "bytes", "serde"] } mpeg2ts = "0.6.0" num_enum = { workspace = true } roxmltree = "0.21.1" +rustc-hash = { workspace = true } scuffle-av1 = { version = "0.1.4" } scuffle-h265 = { version = "0.2.2" } serde = { workspace = true } diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 6f557c3dca7..c3069228fb0 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -39,6 +39,7 @@ use mpeg2ts::ts::{ }; use moq_net::Timestamp; +use rustc_hash::FxHashMap; use super::catalog::Catalog as _; use crate::catalog::Stream; @@ -147,7 +148,7 @@ where /// The program tables changed, so the next frame muxed is a tune-in point that leads with them. revised: bool, /// The next continuity counter per PID, numbered as packets go out. - counters: HashMap, + counters: FxHashMap, /// PMT program-level descriptors captured on import, re-emitted in the PMT. program_descriptors: Vec, /// Transport/service identity captured on import, used to rebuild a consistent @@ -1209,7 +1210,7 @@ where held: None, generation: 0, tracks: HashMap::new(), - counters: HashMap::new(), + counters: FxHashMap::default(), program_descriptors: Vec::new(), program: None, si: BTreeMap::new(), diff --git a/rs/moq-mux/src/container/ts/schedule.rs b/rs/moq-mux/src/container/ts/schedule.rs index f0c2424cb47..d5ca457711d 100644 --- a/rs/moq-mux/src/container/ts/schedule.rs +++ b/rs/moq-mux/src/container/ts/schedule.rs @@ -1,11 +1,12 @@ //! Lays a TS export's packets onto its PCR grid. use std::cmp::Reverse; -use std::collections::{BinaryHeap, HashMap, VecDeque}; +use std::collections::{BinaryHeap, VecDeque}; use std::time::Duration; use anyhow::Context; use mpeg2ts::ts::TsPacket; +use rustc_hash::FxHashMap; use super::export::PCR_INTERVAL; @@ -566,7 +567,7 @@ pub(super) struct Schedule { /// The due slot of the last unit pushed. last: Option, /// Each PID's buffers, from [`Self::set_buffer`]. - buffers: HashMap, + buffers: FxHashMap, /// The PID every slot's clock packet rides on, from [`Self::set_clock`]. clock: Option, /// The PMT's PID, which shares TBsys with the PAT, from [`Self::set_tables`]. @@ -575,12 +576,12 @@ pub(super) struct Schedule { decoding: VecDeque<(u16, u128, usize)>, /// When each transport buffer's last admitted packet finishes leaving it, and when it /// last started filling from empty. - transport: HashMap, + transport: FxHashMap, /// When Bsys has removed every system byte admitted so far, and the Rsys it empties at /// whenever it holds one. system: Option<(i128, u128)>, /// When each video PID's MB finishes passing its last admitted payload on into EB. - mb: HashMap, + mb: FxHashMap, /// Every source has ended, so a unit that misses its deadline is the tail going out late /// rather than an output that cannot keep up. ended: bool, @@ -599,13 +600,13 @@ impl Schedule { next: None, horizon: None, last: None, - buffers: HashMap::new(), + buffers: FxHashMap::default(), clock: None, pmt: None, decoding: VecDeque::new(), - transport: HashMap::new(), + transport: FxHashMap::default(), system: None, - mb: HashMap::new(), + mb: FxHashMap::default(), ended: false, seq: 0, lead: None, @@ -869,8 +870,8 @@ impl Schedule { /// The PIDs the program tables still to go out hold back, each with the first of them that /// does: a unit on one, pushed after those tables, waits for them. `left` is how many /// packets each unit has still to send. - fn holding(&self, left: impl Fn(usize) -> usize) -> HashMap { - let mut held = HashMap::new(); + fn holding(&self, left: impl Fn(usize) -> usize) -> FxHashMap { + let mut held = FxHashMap::default(); for (i, unit) in self.units.iter().enumerate() { let Some(program) = unit.tables.as_ref().filter(|_| left(i) > 0) else { continue; @@ -1105,8 +1106,8 @@ impl Schedule { // One head per PID preserves decode order without walking all future units at every // transport position. Removal events keep occupancy work proportional to units. let mut lanes: Vec> = Vec::new(); - let mut lane_by_pid = HashMap::new(); - let mut occupancy: HashMap = HashMap::new(); + let mut lane_by_pid = FxHashMap::default(); + let mut occupancy: FxHashMap = FxHashMap::default(); let mut removals = Vec::new(); self.decoding.retain(|&(_, decode, _)| decode as i128 > time(0)); for &(pid, decode, held) in &self.decoding { @@ -1432,7 +1433,7 @@ impl Schedule { fn admit(&mut self, index: u128, mut budget: usize) -> Vec { // Unpadded layout has no exact packet clock until the slot is assembled, so it // retains units until their decode instant is behind the opening grid point. - let mut occupancy: HashMap = HashMap::new(); + let mut occupancy: FxHashMap = FxHashMap::default(); self.decoding .retain(|&(_, decode, _)| decode >= index.saturating_sub(1) * PCR_INTERVAL.as_nanos()); for &(pid, _, held) in &self.decoding { @@ -1445,7 +1446,7 @@ impl Schedule { let mut order: Vec = (0..self.units.len()).collect(); order.sort_by_key(|&i| self.units[i].due); let mut take = vec![0; self.units.len()]; - let mut carried: HashMap = HashMap::new(); + let mut carried: FxHashMap = FxHashMap::default(); let mut stopped: Vec = Vec::new(); let mut holding = self.holding(|i| self.units[i].remaining()); for i in order { @@ -1505,6 +1506,8 @@ impl Schedule { #[cfg(test)] mod tests { + use std::collections::HashMap; + use super::*; /// A unit of `count` packets on PID `pid`. From c34b73ffa53e55466b26437b8ce97b9f2ea63171 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 16:50:24 -0700 Subject: [PATCH 11/13] docs(quest): record the final release comparison and its open sparse read slope The 2026-10-25 run left one accidental regression, the sparse read slope from dynamic range requests, and an inconclusive relay fanout pairing. The epic keeps them as remaining work with a child quest, instead of finishing. Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 + quest/m1/api-audit/README.md | 4 + quest/m1/release-audit/README.md | 1 + quest/m1/release-perf/README.md | 221 +++++++++++++++++++++++++ quest/m1/release-perf/sparse-misses.md | 33 ++++ quest/m2/audit-perf.md | 9 + quest/m2/performance-survey/README.md | 9 +- 7 files changed, 274 insertions(+), 4 deletions(-) create mode 100644 quest/m1/release-perf/README.md create mode 100644 quest/m1/release-perf/sparse-misses.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 9fa5a3705e3..84cf6e6c7b1 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -145,6 +145,7 @@ Catalog HLS field. - [WebGPU on Safari](/quest/m1/webgpu-safari.md) - the WebGPU renderer is verified on Safari 26 for macOS and iOS - [JSON encoder options without a downcast](/quest/m1/json-encoder-options.md) - moq-mux's `json_snapshot` takes its `deltas` switch through a typed path any custom catalog entry can use, not an `Any` downcast - [CI housekeeping](/quest/m1/ci-housekeeping.md) - validate queue nextest at 24 threads and remove the actionlint queue ignore after its release +- [Main is no slower than release](/quest/m1/release-perf/README.md) - every benchmark where main is slower than release is fixed or accepted with a reason, checked by a final paired run before the cut - [Drop announce="never"](/quest/m1/publish-announce-never.md) - `@moq/publish` offers only announce modes a watcher can reach - [Drop unused OBS outputs](/quest/m1/obs-output-variants.md) - the plugin registers only `moq_output`, so the broken audio-only variant is gone - [API Extractor checks](/quest/m1/api-extractor-checks.md) - API Extractor fails a PR on a forgotten export, an unresolved declaration, or an undocumented export, replacing the hand-rolled `.d.ts` checkers diff --git a/quest/m1/api-audit/README.md b/quest/m1/api-audit/README.md index 533e165af92..f382640a3e4 100644 --- a/quest/m1/api-audit/README.md +++ b/quest/m1/api-audit/README.md @@ -74,3 +74,7 @@ Rejected in planning: - [One lifecycle vocabulary](/quest/m1/api-audit/lifecycle.md) - finish, abort, closed, and close mean one thing on every handle, and Rust and JS observers fire alike - [moq-net names say one thing](/quest/m1/api-audit/net-names.md) - demand says `wanted`, colliding stats and transport names are renamed, and Rust and JS info lookups mirror - [Request handlers are named requests](/quest/m1/api-audit/requests.md) - `dynamic()` and `Dynamic` become `requests()` and `Requests` at every layer + +## Related + +- [Main is no slower than release](/quest/m1/release-perf/README.md) - the other release-readiness check diff --git a/quest/m1/release-audit/README.md b/quest/m1/release-audit/README.md index 4dd167cc32e..fbd79026a82 100644 --- a/quest/m1/release-audit/README.md +++ b/quest/m1/release-audit/README.md @@ -98,6 +98,7 @@ child records whether its behavior is new on main or inherited from release. - [JS subscription ranges](/quest/m1/subscribe-ranges/js.md) - range lists and frame offsets build on the bounds fixed here - [OBS video source](/quest/m2/obs-moq-video/source.md) - the decoder replacement keeps the timing and EOF behavior fixed here - [Compressed tracks](/quest/m1/compress/README.md) - application migration has an existing owner +- [Measured release performance](/quest/m1/release-perf/README.md) - existing paired benchmark regressions and their confirmation runs - [Transport performance audit](/quest/m2/audit-perf.md) - congestion-control coverage remains outstanding - [Binding timestamps](/quest/m1/binding-timestamps.md) - exact timestamps across language APIs - [Dependency audit](/quest/m2/audit-deps.md) - existing advisory inventory diff --git a/quest/m1/release-perf/README.md b/quest/m1/release-perf/README.md new file mode 100644 index 00000000000..8322189c223 --- /dev/null +++ b/quest/m1/release-perf/README.md @@ -0,0 +1,221 @@ +# Main is no slower than release + +## Goal + +Before the next release cut, every benchmark where `main` is slower than +`release` is either fixed or accepted with a recorded reason. Accidental cost +is fixed. Cost that pays for a feature `main` deliberately added is accepted, +and the reason is written here. + +This epic's own work: once every child has landed, re-run the cases the +2026-10-25 run left open (below) against `release`, and confirm the relay +fanout load. The full run and the browser frame path need no repeat; the +[performance audit](/quest/m2/audit-perf.md) now pairs every later cut with +`release`. + +## Plan + +Decided 2026-10-21, when planning from a paired comparison of `release` +(`0c56e5b4e`, which included the bench backport +[#6103](https://github.com/moq-dev/moq/pull/6103)) against `main` +(`8b0d9952f`). The run used 811 paired Criterion cases, three alternating +confirmation rounds for the largest moves, JS on bun, and the relay load +harness. It ran on a shared 8-core host, so a small wall-time change may be +noise. + +- The epic gates the cut through the [m1 milestone](/quest/m1/README.md), + because `main` must not ship slower than `release` without a decision. +- In scope: the five cases that held through confirmation, plus the large + screen-only moves. The screen-only moves are re-measured with callgrind + instruction counts, and any that hold are fixed. +- Only accidental cost is fixed. Cost from an intended feature is accepted, + and the reason is recorded here. +- The IETF `/fetch` rows of `session_delivery_group_cost` are accepted and + dropped from release pairing. They measure different work on each branch: + release's bare track handle opened a live SUBSCRIBE, while main sends a real + FETCH per viewer ([#5820](https://github.com/moq-dev/moq/pull/5820) made + handles lazy, and [#4974](https://github.com/moq-dev/moq/pull/4974) added + fetch-only demand). The lite `/fetch` cost was real: a relay sent one + upstream FETCH per group, but revived the parked logical track for every + viewer's FETCH. [#6286](https://github.com/moq-dev/moq/pull/6286) serves a + whole cached group from the parked copy instead, cutting 12 to 19% of the + instructions per viewer where viewers share a group. The rest of the gap to + a viewer that holds a query is accepted: each viewer's own fetch of a group + it lacks revives its logical track, so the open group keeps its route and + fails over mid-read, and its copy releases the route's demand once idle. +- `stats/session-report` is accepted at O(live sessions), after a cheap fix. + Per-label payload bytes ([#5731](https://github.com/moq-dev/moq/pull/5731)) + match payload billing per label, such as per signing key. After removing + redundant row locking and live-session CAS, three alternating measurements + give 52.2 us for 10,000 sessions without wires and 388.4 us with one real + transport wire each. A once-per-second drain uses about 0.04% of a core + with those wires; the earlier under-0.01% estimate omitted their walk. + This remaining per-session and per-connection sampling is accepted. Deleting + per-label bytes was rejected. +- `track_gc/idle` is accepted at about 20 ns, flat across track counts, after + [#6291](https://github.com/moq-dev/moq/pull/6291) skipped the clock read + with no wall deadline and reused the bench's waiter (36.7 ns before). The + remaining ~8 ns over release's 11.6 ns is the lock and driver registration + that the wall-clock aging from + [#5548](https://github.com/moq-dev/moq/pull/5548) needs. An earliest-deadline + atomic cannot prove a replacement driver is registered, and a lock-free + notification primitive is not worth ~8 ns. +- `track_gap_flush` is accepted at O(log gaps + retired entries) per shared + successor, after [#6287](https://github.com/moq-dev/moq/pull/6287) indexed + the gap ledger and retired a shared verdict's gaps with one split. Each + retired entry must be freed exactly once, so the batch is already amortized + optimal; a representation that hides that reclamation would widen scope for + no measured benefit. Sparse arrivals cost O(log gaps) amortized, and at 4096 + gaps flush, read, and expiry instructions fell 80 to 95%. +- The mux exports' and Annex B import's fixed cost per broadcast is accepted. + It publishes required output: a per-track timeline and an immediately + completed catalog record. Their cold DEFLATE state is most of it (81% of a + one-frame import's instructions are its zero-fill), and the encoders already + build that state lazily on first use. It is paid once per broadcast: the + import ratio falls from 3.97x at one frame to 1.49x at 64. TS export's + growth with track count is its T-STD schedule (see the final run). +- The model's remaining per-group and per-frame cost is accepted, after + [moq-dev/moq#6294](https://github.com/moq-dev/moq/pull/6294) cut the + accidental part (a second track upgrade per first frame, a 128-bit division + per buffered datagram on join, and draining empty waiter lists on every + write). What remains pays for features: the arrival stamp each frame reads + from the clock ([#6249](https://github.com/moq-dev/moq/pull/6249)), largest + notifications ([#5725](https://github.com/moq-dev/moq/pull/5725)), the + rewind check and starts index each first frame fills + ([#5803](https://github.com/moq-dev/moq/pull/5803), + [#6074](https://github.com/moq-dev/moq/pull/6074)), indexed expiry wakes, + the bounded wall-clock expiry scan per append, and a subscriber's ranges and + datagram max delay ([#5195](https://github.com/moq-dev/moq/pull/5195)). + None of it grows with the sweeps. `track_stamp_groups/sparse` stays highest + at 4.4x, because release's first frame skipped nearly all of that work. +- Session delivery is accepted at 1.28x to 1.77x in instructions. It is the + per-group and per-frame cost above on every hop, measured through the + flow-controlled mock transport + ([#5530](https://github.com/moq-dev/moq/pull/5530)) that release's harness + lacked. + +### Release as a baseline + +`release` lacks many of the features the backported benches exercise: +per-reader delivery records, wall-clock aging, the gap ledger, rewind checks, +and epochs. So a ratio against `release` shows where to look, not what to fix. +Main's own sweeps over a size parameter are the better regression signal: a +cost that grows with the sweep is accidental, while a flat per-operation cost +usually pays for a feature. + +### Confirmed (three rounds, ratio is main / release) + +| Bench | release | main | Ratio | Owner | +|---|---|---|---|---| +| `track_gap_flush/4096` | 5.7 us | 551 us | 96x | accepted (above) | +| `track_gap_flush/512` | 1.8 us | 48 us | 27x | accepted (above) | +| `track_gap_flush/64` | 1.5 us | 6.2 us | 4.1x | accepted (above) | +| `stats/session-report/1000x10` | 11.0 us | 86.8 us | 7.9x | accepted (above) | +| `track_gc/idle/16384` | 11.6 ns | 73.7 ns | 6.3x | accepted (above) | +| `origin/resume_overlap/sequence/1r_64g_32c` | 93 us | 426 us | 4.6x | fixed in [moq-dev/moq#6292](https://github.com/moq-dev/moq/pull/6292) | +| `session_delivery_group_cost/moq-transport-22/.../viewers=64/.../fetch` | 1.19 ms | 3.86 ms | 3.2x | accepted (above) | +| `group_write_frames/single/8192` | 424 us | 486 us | 1.17x | accepted (above) | + +Main is much faster on subscriber churn, joins, range demand, and idle +sessions (by 100x to 3000x). The relay load test used 21% less CPU on video +and 12% less on fanout, with the same delivery. + +### Screen only, re-measured + +Re-measured on 2026-10-22 with callgrind instructions per operation, release +rebuilt under main's dev shell with fat LTO. Ratios are main over release; +the moq-net and kio rows include the fixes above. + +| Bench | Screen | Instructions | Owner | +|---|---|---|---| +| `container-export/ts/4x1`, `mkv/4x1` | 4.7x, 3.9x | 1.70x, 1.59x | accepted (above) | +| `annexb-import`, 1 and 64 frames | median 2.1x | 3.97x, 1.49x | accepted (above) | +| `track_stamp_groups/append/64`, `sparse/64` | median 2.7x | 2.21x, 4.43x | accepted (above) | +| `track_parallel_write/1` | 1.6x | 1.99x | accepted (above) | +| `track_fanout_frame/1` | 1.42x | 1.86x | accepted (above) | +| `track_empty_run/{fresh/1,stale/8}/512` | median 1.36x | 1.53x | accepted (above) | +| `track_datagram_subscriber_join/0d/1`, `64d/64` | median 2.3x | 2.12x, 3.55x | accepted (above) | +| `track_sparse_read/timed/4096` | 4.3x | | fixed in [moq-dev/moq#6287](https://github.com/moq-dev/moq/pull/6287), regressed again (final run) | +| `origin/announce_duplicate`, `1d_1s` to `240d_240s` | 4.3x | 1.06x to 1.16x | noise | +| `session_delivery_{relays,paced,scale}` | 1.43x to 1.56x | 1.28x to 1.77x | accepted (above) | +| moq-stats `aggregate`, live=16, departed=4096 | 2.1x | 1.05x | noise | +| kio `contended_{notify,log}`, 16 threads | 4x to 5x | wall 1.04x to 1.15x | noise | +| JS `net/bench/frames.ts` on bun | 1.2x to 2.2x | | accepted (below) | + +The kio contention benches spin across threads, so instructions don't measure +them; the row is the least loaded of three alternating wall-time rounds, and +later rounds swung both ways. + +JS `net/bench/frames.ts` on bun had lite rows at 1.2 to 1.3x and IETF rows at +1.3 to 2.2x. Fixed in Chromium: +[#6296](https://github.com/moq-dev/moq/pull/6296) removed a retained-history +scan, and [#6479](https://github.com/moq-dev/moq/pull/6479) stopped +releasing a finished stream's reader lock, which built a `TypeError` per +group stream. Every row then sits within the A/A spread of release. +The group write/read microbench stays 3 to 12% above release against an +A/A of 3 to 8% on the same host. Decided 2026-10-23 (maintainer): browser +parity accepted on this evidence; the 2026-10-25 run confirmed it. + +### Final run (2026-10-25) + +`release` (`3db33f6715`) against `main` (`018f1c46e5`): 1,019 paired +Criterion cases, three relay load pairings, and nine Chromium frame rounds +with an A/A pair, at host load 20 to 60. Each slower case was re-run in +alternating rounds and counted in callgrind instructions. + +Fixed in [moq-dev/moq#6784](https://github.com/moq-dev/moq/pull/6784): + +- The MKV and FLV exports rebuilt the jitter clock's expected track set on + every frame; they now set it once per catalog update. + `container-export/mkv/16x64` went from 27.1M to 17.6M instructions (2.27x to + 1.47x release, flat across the sweep), `flv/16x64` 14.5M to 12.9M. +- The TS export and its T-STD schedule hash PIDs with FxHash: + `container-export/ts/16x64` 30.8M to 25.5M (2.70x to 2.24x). +- Benches that broke since release now run: the moq-mux catalog bench caps its + renditions, the relay auth bench reads its observed state directly, moq-stats + `producer_tick` skips tiers above one at 50,000 paths, and `bench/run.sh` + skips targets that are not Criterion. + +Accepted, flat per operation: + +- e2ee `protect` is 1.5x release on grouped frames and 1.21x on datagrams. The + crypto is unchanged; the rest is the per-group model cost accepted above. + Both branches expand the AES-GCM key on every frame, about a quarter of + `protect`, which the [performance audit](/quest/m2/audit-perf.md) owns. +- gst `moqsink/eos` is +12% instructions per event, from + [#5512](https://github.com/moq-dev/moq/pull/5512) and the gstreamer-rs 0.25 + bump in [#6169](https://github.com/moq-dev/moq/pull/6169). +- `group_write_frames` (1.67x) and `track_stamp_groups/sparse` (1.48x, down + from 4.4x) are the per-frame and per-group costs accepted above, as are the + session join, burst, and delivery families (1.2x to 1.76x). +- The TS export still grows with track count (1.50x at `1x1` to 2.24x at + `16x64`). The T-STD schedule scans and sorts every queued unit per PCR slot, + which the [performance audit](/quest/m2/audit-perf.md) owns. + +Noise: `frame_recv/upfront/size=262144/streams=64` is 3 to 4x in wall time at +0.96x instructions. Its 1.6M page faults (against 192k) are glibc returning +the freed 256 KiB buffers between iterations; with +`GLIBC_TUNABLES=glibc.malloc.mmap_threshold=1048576:glibc.malloc.trim_threshold=1073741824` +both sides fault 4.6k times and time the same, and the relay runs jemalloc. +`session_idle` IETF at 1024 sessions, `origin/resume_overlap`, +`origin/retire`, kio `contended_notify`, and moq-stats `producer/merge` are +at parity in alternating rounds or instructions. + +Relay load: video used 7.8% less relay CPU with the same delivery. Fanout used +16.4% more, but its paired deltas were -1.5%, +16.4%, and +26.1% (MAD 9.7%) +at load 25, and the 2026-10-21 run measured 12% less, so it needs a quiet +re-run. Browser frames: lite rows 1.09 to 1.14x and IETF 0.95x, the same as +`main` at the 2026-10-23 decision (1.13x lite). + +Open: the sparse read slope from +[dynamic range requests](https://github.com/moq-dev/moq/pull/6171), owned by +the child below, and the fanout re-run. + +## Required + +- [Sparse reads judge misses incrementally](/quest/m1/release-perf/sparse-misses.md) - `track_sparse_read` stays flat across gaps, as before range requests + +## Related + +- [Performance audit](/quest/m2/audit-perf.md) - compares nightly to nightly; this epic compares main to release +- [Calibrate the paired nightly benchmarks](/quest/m1/bench-ci.md) - noise floor of the paired nightly-to-nightly comparison on one runner diff --git a/quest/m1/release-perf/sparse-misses.md b/quest/m1/release-perf/sparse-misses.md new file mode 100644 index 00000000000..7ed3dc13670 --- /dev/null +++ b/quest/m1/release-perf/sparse-misses.md @@ -0,0 +1,33 @@ +# [M] Sparse reads judge misses incrementally + +## Goal + +A reader of a sparse track pays O(log gaps) per group handed out, as it did +before [dynamic range requests](https://github.com/moq-dev/moq/pull/6171). +`track_sparse_read` and the setup of `track_gap_flush` stay flat across their +gap sweeps. + +## Plan + +Found by the 2026-10-25 main against release run. Every hand-out goes through +`Cursor::probe` (`model/track.rs`), and `miss::Misses::judge` +(`model/track/miss.rs`) rebuilds its clipped span from the whole missable +window and re-judges every deferred gap, so each read walks every gap below +it. Instructions per write and read pair of `track_sparse_read/timed`: 152k +at 16 gaps, 2.57M at 256, and 51.4M at 4096, which is 1079x release in wall +time (2.75 ms against 2.5 us). The untimed rows match. `track_gap_flush` +builds its gaps through the same path, so its untimed setup is quadratic +(50.8G instructions at 4096 gaps). + +Judge only what changed since the last probe: the groups the read passed, +gaps the ledger resolved, and a window or budget change. A probe with +nothing new must not touch the deferred set. Keep the fills and drops +that range requests serve; the sweep, not a release ratio, is the acceptance signal. +[Drop set](/quest/m1/drop-set.md) reshapes the same state, so whichever lands +second rebases onto the other. + +Public API: none. Wire: none. + +## Related + +- [Drop set](/quest/m1/drop-set.md) - per-cursor miss judgement over the same ledger diff --git a/quest/m2/audit-perf.md b/quest/m2/audit-perf.md index bebfbdc39f1..b0313979e76 100644 --- a/quest/m2/audit-perf.md +++ b/quest/m2/audit-perf.md @@ -112,6 +112,12 @@ did not move. catalog on every estimate to validate it before retaining it (62% of `catalog_estimate_fanout/hang/tracks_4/4`). Validating only the edited entry would weaken that guarantee for custom configs, so it needs a decision. +- The TS export grows with track count (1.50x release at `container-export/ts/1x1`, + 2.24x at `16x64`): `Schedule::index` and `Schedule::admit` + (`moq-mux/src/container/ts/schedule.rs`) scan and sort every queued unit + per PCR slot. +- e2ee expands the AES-GCM key on every frame (`aead` in + `moq-e2ee/src/protect.rs`), about a quarter of `protect`; release does too. ### Method: attribute with instruction counts, not wall time @@ -131,6 +137,9 @@ wall-clock bisect blames the wrong commit. Count instructions instead: `finish` for `track_stamp_groups`) and divide by the count. - `callgrind_annotate --inclusive=yes` diffs between two commits point at the step; workspace sources show as `${workspace}`, so read them by name. +- A wall-time hold at equal instructions may be page faults: compare + `/usr/bin/time -f %R`, then pin glibc with `GLIBC_TUNABLES` (see + [the m1 release run](/quest/m1/release-perf/README.md)) before blaming code. - A bench that is not CPU-bound is not attributable this way: `relay_auth_refresh` runs real loopback sockets and moved +27-60% on CI with +0.4% instructions. diff --git a/quest/m2/performance-survey/README.md b/quest/m2/performance-survey/README.md index 95f5d0adbb9..05427945cc7 100644 --- a/quest/m2/performance-survey/README.md +++ b/quest/m2/performance-survey/README.md @@ -143,10 +143,11 @@ No duplicate quest or workflow edit belongs here. ### Existing ownership and deliberately deferred work -- The [performance audit](/quest/m2/audit-perf.md) repeats the main against - release comparison before each cut. The m1 comparison already fixed or - accepted sparse gaps, resume duplicates, idle GC, stats drain, FETCH fan-out, - JS frame regressions, and paired comparison correctness. +- [Release performance](/quest/m1/release-perf/README.md) owns the main + against release comparison for this cut, including the sparse read slope; + it already fixed or accepted resume duplicates, idle GC, stats drain, FETCH + fan-out, JS frame regressions, and paired comparison correctness. The + [performance audit](/quest/m2/audit-perf.md) repeats it before later cuts. - [Perf](/quest/m2/perf/README.md) owns measured channel contention, relay allocations, ingest batching, worker quiescence and runtime selection. - The [release audit](/quest/m1/release-audit/README.md) bounded the media From 3d09a5254fa2e62efd76d84ddcd3a7ca5171fa20 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 17:07:07 -0700 Subject: [PATCH 12/13] docs(quest): record the release comparison decisions Co-Authored-By: Claude Opus 5.5 --- quest/m1/release-perf/README.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/quest/m1/release-perf/README.md b/quest/m1/release-perf/README.md index 8322189c223..7bf7d3211b8 100644 --- a/quest/m1/release-perf/README.md +++ b/quest/m1/release-perf/README.md @@ -7,9 +7,9 @@ Before the next release cut, every benchmark where `main` is slower than is fixed. Cost that pays for a feature `main` deliberately added is accepted, and the reason is written here. -This epic's own work: once every child has landed, re-run the cases the -2026-10-25 run left open (below) against `release`, and confirm the relay -fanout load. The full run and the browser frame path need no repeat; the +This epic's own work: once every child has landed, confirm the sparse rows +and the relay fanout load against `release` (see the 2026-10-25 decisions). +The browser frame path needs no repeat; the [performance audit](/quest/m2/audit-perf.md) now pairs every later cut with `release`. @@ -207,9 +207,19 @@ at load 25, and the 2026-10-21 run measured 12% less, so it needs a quiet re-run. Browser frames: lite rows 1.09 to 1.14x and IETF 0.95x, the same as `main` at the 2026-10-23 decision (1.13x lite). -Open: the sparse read slope from -[dynamic range requests](https://github.com/moq-dev/moq/pull/6171), owned by -the child below, and the fanout re-run. +Decided 2026-10-25 (maintainer), in +[moq-dev/moq#6784](https://github.com/moq-dev/moq/pull/6784): + +- The sparse read slope from + [dynamic range requests](https://github.com/moq-dev/moq/pull/6171) is the + child below, and it blocks the cut. +- The TS schedule's growth is left to the + [performance audit](/quest/m2/audit-perf.md); it does not block the cut. +- Once the child lands, re-run the relay fanout pairing on a quiet host, + recording `/proc/loadavg`, with + `BENCH_ARTIFACTS= just bench origin/release` (it also re-checks + the sparse rows). Fix it if the relay + CPU still rises with consistent sign; otherwise this epic is done. ## Required From 01e3bab249a1e6dec6e6a040ecc4c34b96a2960d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 17:11:45 -0700 Subject: [PATCH 13/13] docs(quest): account for the dash relay and reprice rows Co-Authored-By: Claude Opus 5.5 --- quest/m1/release-perf/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/quest/m1/release-perf/README.md b/quest/m1/release-perf/README.md index 7bf7d3211b8..1b55953452a 100644 --- a/quest/m1/release-perf/README.md +++ b/quest/m1/release-perf/README.md @@ -188,6 +188,10 @@ Accepted, flat per operation: - `group_write_frames` (1.67x) and `track_stamp_groups/sparse` (1.48x, down from 4.4x) are the per-frame and per-group costs accepted above, as are the session join, burst, and delivery families (1.2x to 1.76x). + `session_dash_relays` is the same per-track cost: 1.50x instructions at 768 + tracks over 2 or 3 relays, and 0.63x over 33 relays, so it falls with relay + count. The import fixed costs above (`annexb-import`, `cmaf-import` at one + frame) and the IETF `/fetch` delivery rows also reappear unchanged. - The TS export still grows with track count (1.50x at `1x1` to 2.24x at `16x64`). The T-STD schedule scans and sorts every queued unit per PCR slot, which the [performance audit](/quest/m2/audit-perf.md) owns. @@ -198,8 +202,9 @@ the freed 256 KiB buffers between iterations; with `GLIBC_TUNABLES=glibc.malloc.mmap_threshold=1048576:glibc.malloc.trim_threshold=1073741824` both sides fault 4.6k times and time the same, and the relay runs jemalloc. `session_idle` IETF at 1024 sessions, `origin/resume_overlap`, -`origin/retire`, kio `contended_notify`, and moq-stats `producer/merge` are -at parity in alternating rounds or instructions. +`origin/retire`, `origin/reprice_duplicate` (0.55x instructions), kio +`contended_notify`, and moq-stats `producer/merge` are at parity in +alternating rounds or instructions. Relay load: video used 7.8% less relay CPU with the same delivery. Fanout used 16.4% more, but its paired deltas were -1.5%, +16.4%, and +26.1% (MAD 9.7%)