From 1bc532f509ef54ec7c88a37e426497a86e59c6db Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:22:57 -0700 Subject: [PATCH 1/5] quest: claim bench-pin From 3c3bd8820a2c65bab7a93b974653881db8416f29 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:24:17 -0700 Subject: [PATCH 2/5] bench: pin hybrid-CPU runs to the performance cores Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 13 ++++++++++--- bench/run.sh | 16 ++++++++++++++-- flake.nix | 18 +++++++++++------- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/bench/README.md b/bench/README.md index a6424ad7ba4..ad3e50714ba 100644 --- a/bench/README.md +++ b/bench/README.md @@ -66,8 +66,8 @@ nix develop --command just bench-runtime 5 16 ``` Runtime comparison requires Linux because io\_uring and relay process metrics -come from Linux interfaces. The default worker count is the number of online -logical CPUs. +come from Linux interfaces. The default worker count is the number of logical +CPUs the run may use. Sweep the relay's QUIC shard count against the connection count, for the room (`video`: every connection publishes and subscribes) and `fanout` (one @@ -78,7 +78,7 @@ nix develop --command just bench-shard nix develop --command just bench-shard "0 1 8 24" "64 512" ``` -Zero shards is the shared runtime with one Tokio thread per logical CPU; any +Zero shards is the shared runtime with one Tokio thread per usable CPU; any other count is that many unpinned Tokio workers. The table reports CPU per connection, so a steering, dispatch, or per-shard cost that grows with either axis reads as a slope. Linux only, since nothing else steers a reuseport @@ -145,6 +145,13 @@ work. Benchmark output is informational and machine-specific. Crashes, zero delivery, and invalid samples still fail the command. +On a hybrid CPU the same case measures far apart on performance and efficiency +cores, so `run.sh` pins itself, and every build and benchmark process it starts, +to the performance cores wherever the kernel lists them +(`/sys/devices/cpu_core/cpus` on Intel hybrid parts). The run prints the pinned +cores beside its result and records them as `affinity` in the metadata. A CPU +with one core type runs unpinned. + BASE comparisons run paired rounds (default three, `MOQ_BENCH_RUNTIME_ROUNDS`), alternating base/current order. Startup remains outside the final five-second throughput window. `MOQ_BENCH_ARTIFACTS` chooses the evidence parent (default diff --git a/bench/run.sh b/bench/run.sh index 65bbe4b9d0c..08eb1e53c1e 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -17,6 +17,15 @@ if [[ ! $ROUNDS =~ ^[0-9]+$ ]] || ((10#$ROUNDS < 1)); then exit 1 fi ROUNDS=$((10#$ROUNDS)) + +# One case measures far apart on performance and efficiency cores, so on a +# hybrid CPU pin this shell, and with it every build and benchmark process, to +# the performance cores. The kernel only lists them on Intel hybrid parts. +PINNED= +if [[ -r /sys/devices/cpu_core/cpus ]]; then + PINNED=$(/dev/null +fi case $MODE in --audio) BASE= @@ -43,7 +52,7 @@ case $MODE in BASE= RUNTIME_WORKERS=${MOQ_BENCH_RUNTIME_WORKERS:-} if [[ -z $RUNTIME_WORKERS ]]; then - RUNTIME_WORKERS=$(getconf _NPROCESSORS_ONLN) + RUNTIME_WORKERS=$(nproc) if ((RUNTIME_WORKERS > 256)); then RUNTIME_WORKERS=256 fi @@ -115,6 +124,9 @@ cleanup() { exec 3>&- 4>&- fi printf 'Benchmark evidence retained: %s\n' "$RUN" + if [[ -n $PINNED ]]; then + printf 'Pinned to performance cores: %s\n' "$PINNED" + fi } trap cleanup EXIT trap 'exit 130' INT @@ -522,7 +534,7 @@ run_shard_sweep() { HOST_BIN=$CURRENT_TARGET/release/moq-bench-host local cpus workload connections shards label - cpus=$(getconf _NPROCESSORS_ONLN) + cpus=$(nproc) for workload in video fanout; do for connections in $SHARD_CONNECTIONS; do for shards in $SHARD_COUNTS; do diff --git a/flake.nix b/flake.nix index 6cedf2231bd..24817a0ab9c 100644 --- a/flake.nix +++ b/flake.nix @@ -252,13 +252,17 @@ # Developer workflow tooling not needed for builds: jq reads # `cargo metadata` in `just rs check-changed`. - devTools = with pkgs; [ - jq - # Runs the moq.sh installer tests under a strict POSIX shell. - dash - # Proves the temporary bench relay owns its TCP and UDP sockets. - lsof - ]; + devTools = + with pkgs; + [ + jq + # Runs the moq.sh installer tests under a strict POSIX shell. + dash + # Proves the temporary bench relay owns its TCP and UDP sockets. + lsof + ] + # taskset pins bench runs to the performance cores of a hybrid CPU. + ++ lib.optionals (!stdenv.hostPlatform.isDarwin) [ util-linux ]; # Linters / formatters used by `just check` and `just fix`, which # skip a module whose tools aren't on $PATH. CI sets MOQ_STRICT=1, From 19a7974a917bf7edfca22aa19b48144700fac7c5 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:33:33 -0700 Subject: [PATCH 3/5] quest: finish bench-pin Co-Authored-By: Claude Opus 5.5 --- quest/m1/quic/coverage/README.md | 1 - quest/m1/quic/coverage/bench-pin.md | 20 -------------------- 2 files changed, 21 deletions(-) delete mode 100644 quest/m1/quic/coverage/bench-pin.md diff --git a/quest/m1/quic/coverage/README.md b/quest/m1/quic/coverage/README.md index 41f9854a134..953a6b3b3fc 100644 --- a/quest/m1/quic/coverage/README.md +++ b/quest/m1/quic/coverage/README.md @@ -45,7 +45,6 @@ and the harness defaults to Cubic while production runs BBR3. - [Characterization tests](/quest/m1/quic/coverage/characterize.md) - golden scheduling order, and every uncovered frame, reset, limit, and endpoint path, on today's harness - [Plaintext fuzz crypto](/quest/m1/quic/coverage/fuzz-crypto.md) - a fuzz-only plaintext session so the connection target skips TLS and its injected packets decrypt - [Richer simulation workloads](/quest/m1/quic/coverage/sim-workloads.md) - resets, bidi streams, datagrams, and migration in the random workloads, each with its own invariant -- [Pin benchmark cores](/quest/m1/quic/coverage/bench-pin.md) - local base/head comparisons run on one core type on hybrid CPUs ## Related diff --git a/quest/m1/quic/coverage/bench-pin.md b/quest/m1/quic/coverage/bench-pin.md deleted file mode 100644 index e3a481044a8..00000000000 --- a/quest/m1/quic/coverage/bench-pin.md +++ /dev/null @@ -1,20 +0,0 @@ -# [XS] Pin local benchmark comparisons to one core type - -## Goal - -On a hybrid CPU (performance and efficiency cores), `bench/run.sh` runs -both sides of a comparison on the same core type, so a base/head delta no -longer depends on where the scheduler put each run. On a CPU with one core -type nothing changes. - -## Plan - -Decided 2026-10-16 with the maintainer. Seen while landing the moq-quic -benchmarks: on an 8P/16E host the same unpinned case measured 420 µs or -690 µs depending on the core it ran on. Detect the performance cores where -the kernel reports them (for example `/sys/devices/cpu_core/cpus` on Intel -hybrid parts) and pin each benchmark process to them with `taskset`. Print -which cores were used beside the comparison, so a reader knows the run was -pinned. - -Public API: none. Wire: none. From 5048a51ba5af7f5a53ac653b0663997dd3e08cae Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:46:29 -0700 Subject: [PATCH 4/5] bench: report the applied pin mask and ignore OpenMP CPU limits Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 6 +++--- bench/run.sh | 18 +++++++++++++----- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/bench/README.md b/bench/README.md index ad3e50714ba..60e214c66e5 100644 --- a/bench/README.md +++ b/bench/README.md @@ -148,9 +148,9 @@ and invalid samples still fail the command. On a hybrid CPU the same case measures far apart on performance and efficiency cores, so `run.sh` pins itself, and every build and benchmark process it starts, to the performance cores wherever the kernel lists them -(`/sys/devices/cpu_core/cpus` on Intel hybrid parts). The run prints the pinned -cores beside its result and records them as `affinity` in the metadata. A CPU -with one core type runs unpinned. +(`/sys/devices/cpu_core/cpus` on Intel hybrid parts), within any cpuset the run +is confined to. The run prints the pinned cores beside its result and records +them as `affinity` in the metadata. A CPU with one core type runs unpinned. BASE comparisons run paired rounds (default three, `MOQ_BENCH_RUNTIME_ROUNDS`), alternating base/current order. Startup remains outside the final five-second diff --git a/bench/run.sh b/bench/run.sh index 08eb1e53c1e..09982121f34 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -20,12 +20,20 @@ ROUNDS=$((10#$ROUNDS)) # One case measures far apart on performance and efficiency cores, so on a # hybrid CPU pin this shell, and with it every build and benchmark process, to -# the performance cores. The kernel only lists them on Intel hybrid parts. +# the performance cores. The kernel only lists them on Intel hybrid parts, and +# intersects the request with any cpuset, so report the mask it applied. PINNED= if [[ -r /sys/devices/cpu_core/cpus ]]; then - PINNED=$(/dev/null + if ! PINNED=$(taskset -pc "$(&2 + exit 1 + fi fi + +# nproc counts the allowed CPUs, unless OpenMP variables override it. +usable_cpus() { + env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc +} case $MODE in --audio) BASE= @@ -52,7 +60,7 @@ case $MODE in BASE= RUNTIME_WORKERS=${MOQ_BENCH_RUNTIME_WORKERS:-} if [[ -z $RUNTIME_WORKERS ]]; then - RUNTIME_WORKERS=$(nproc) + RUNTIME_WORKERS=$(usable_cpus) if ((RUNTIME_WORKERS > 256)); then RUNTIME_WORKERS=256 fi @@ -534,7 +542,7 @@ run_shard_sweep() { HOST_BIN=$CURRENT_TARGET/release/moq-bench-host local cpus workload connections shards label - cpus=$(nproc) + cpus=$(usable_cpus) for workload in video fanout; do for connections in $SHARD_CONNECTIONS; do for shards in $SHARD_COUNTS; do From 87e4d6fd83b5bb14a18adac009133f50649015d7 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 20:21:09 -0700 Subject: [PATCH 5/5] bench: keep a narrower inherited affinity when pinning Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 4 ++-- bench/run.sh | 23 +++++++++++++++++------ flake.nix | 18 +++++++----------- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/bench/README.md b/bench/README.md index 60e214c66e5..404dc70680e 100644 --- a/bench/README.md +++ b/bench/README.md @@ -148,8 +148,8 @@ and invalid samples still fail the command. On a hybrid CPU the same case measures far apart on performance and efficiency cores, so `run.sh` pins itself, and every build and benchmark process it starts, to the performance cores wherever the kernel lists them -(`/sys/devices/cpu_core/cpus` on Intel hybrid parts), within any cpuset the run -is confined to. The run prints the pinned cores beside its result and records +(`/sys/devices/cpu_core/cpus` on Intel hybrid parts), within the CPUs it was +already allowed, so an outer `taskset` or cpuset still narrows it. The run prints the pinned cores beside its result and records them as `affinity` in the metadata. A CPU with one core type runs unpinned. BASE comparisons run paired rounds (default three, `MOQ_BENCH_RUNTIME_ROUNDS`), diff --git a/bench/run.sh b/bench/run.sh index 09982121f34..c0d063c6122 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -20,14 +20,25 @@ ROUNDS=$((10#$ROUNDS)) # One case measures far apart on performance and efficiency cores, so on a # hybrid CPU pin this shell, and with it every build and benchmark process, to -# the performance cores. The kernel only lists them on Intel hybrid parts, and -# intersects the request with any cpuset, so report the mask it applied. +# the performance cores it may already use. The kernel only lists them on Intel +# hybrid parts. PINNED= if [[ -r /sys/devices/cpu_core/cpus ]]; then - if ! PINNED=$(taskset -pc "$(&2 - exit 1 - fi + PINNED=$( + python3 - "$$" <<'EOF' +import os, sys +pid = int(sys.argv[1]) +cores = set() +for part in open("/sys/devices/cpu_core/cpus").read().strip().split(","): + first, _, last = part.partition("-") + cores.update(range(int(first), int(last or first) + 1)) +allowed = cores & os.sched_getaffinity(pid) +if not allowed: + sys.exit("cannot pin to the performance cores: none are in the allowed CPU set") +os.sched_setaffinity(pid, allowed) +print(",".join(map(str, sorted(allowed)))) +EOF + ) fi # nproc counts the allowed CPUs, unless OpenMP variables override it. diff --git a/flake.nix b/flake.nix index 24817a0ab9c..6cedf2231bd 100644 --- a/flake.nix +++ b/flake.nix @@ -252,17 +252,13 @@ # Developer workflow tooling not needed for builds: jq reads # `cargo metadata` in `just rs check-changed`. - devTools = - with pkgs; - [ - jq - # Runs the moq.sh installer tests under a strict POSIX shell. - dash - # Proves the temporary bench relay owns its TCP and UDP sockets. - lsof - ] - # taskset pins bench runs to the performance cores of a hybrid CPU. - ++ lib.optionals (!stdenv.hostPlatform.isDarwin) [ util-linux ]; + devTools = with pkgs; [ + jq + # Runs the moq.sh installer tests under a strict POSIX shell. + dash + # Proves the temporary bench relay owns its TCP and UDP sockets. + lsof + ]; # Linters / formatters used by `just check` and `just fix`, which # skip a module whose tools aren't on $PATH. CI sets MOQ_STRICT=1,