From 1095e393df0b6756e9c2335eabe1ea03dace2ba2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 22:41:42 -0700 Subject: [PATCH 1/4] chore(quest): claim quest/m0/nightly/ts-schedule.md Quest-Claim: 399f71e16f2cca7b From 183ff97d26f488b929e1226f09818bf51d49f5e2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 22:58:38 -0700 Subject: [PATCH 2/4] ci: temporary ARM diagnostic for the ts_schedule bench delta Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ts-schedule-diag.yml | 73 ++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 .github/workflows/ts-schedule-diag.yml diff --git a/.github/workflows/ts-schedule-diag.yml b/.github/workflows/ts-schedule-diag.yml new file mode 100644 index 00000000000..6de16687132 --- /dev/null +++ b/.github/workflows/ts-schedule-diag.yml @@ -0,0 +1,73 @@ +# Temporary diagnostic for quest/m0/nightly/ts-schedule.md; removed before the PR leaves draft. +name: ts-schedule-diag +on: + push: + branches: [quest/m0/nightly/ts-schedule] + paths: [.github/workflows/ts-schedule-diag.yml] +permissions: + contents: read +jobs: + diag: + runs-on: ubuntu-24.04-arm + timeout-minutes: 50 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - name: Tools + run: sudo apt-get update -q && sudo apt-get install -y -q strace jq + - name: Build + run: | + set -euo pipefail + for side in base:4381af6f07e32d594036723f80d298b67e60085f current:0513c07369357ae0ecd6109d0f6afc61d16b597f; do + name=${side%%:*} rev=${side#*:} + git worktree add --detach "/tmp/$name" "$rev" + (cd "/tmp/$name" && CARGO_TARGET_DIR=/tmp/t-$name cargo bench --locked -p moq-mux --bench ts_schedule --no-run --message-format=json-render-diagnostics) | + jq -r 'select(.reason == "compiler-artifact" and .executable != null and .target.name == "ts_schedule") | .executable' >"/tmp/bin-$name" + cat "/tmp/bin-$name" + done + nproc; lscpu | head -20; getconf PAGESIZE; ldd --version | head -1 + - name: Timing + run: | + set -uo pipefail + for round in 1 2; do + for case_ in 1/1 4/1 16/1 1/64; do + for name in base current; do + printf '%s %s r%s ' "$name" "$case_" "$round" + (cd "/tmp/$name/rs/moq-mux" && CRITERION_HOME=/tmp/crit/$name/$round "$(cat /tmp/bin-$name)" --bench --warm-up-time 1 --measurement-time 2 --sample-size 30 --noplot "ts_schedule/$case_" --exact 2>&1 | grep -E "time:") + done + done + done + - name: Syscalls and faults + run: | + set -uo pipefail + for case_ in 1/1 4/1 16/1; do + for name in base current; do + echo "== $name $case_" + (cd "/tmp/$name/rs/moq-mux" && CRITERION_HOME=/tmp/crit/$name/sys strace -f -c -e trace=brk,mmap,munmap,madvise,mremap -o /tmp/strace.txt "$(cat /tmp/bin-$name)" --bench --profile-time 2 "ts_schedule/$case_" --exact >/dev/null 2>&1) + cat /tmp/strace.txt + (cd "/tmp/$name/rs/moq-mux" && CRITERION_HOME=/tmp/crit/$name/sys /usr/bin/time -f "faults minor=%R major=%F" "$(cat /tmp/bin-$name)" --bench --profile-time 2 "ts_schedule/$case_" --exact 2>&1 >/dev/null | grep faults) + done + done + - name: Allocator held fixed + run: | + set -uo pipefail + for tunables in glibc.malloc.trim_threshold=1073741824:glibc.malloc.mmap_threshold=33554432 glibc.malloc.mmap_threshold=131072; do + for case_ in 1/1 4/1 16/1; do + for name in base current; do + printf '%s %s %s ' "$tunables" "$name" "$case_" + (cd "/tmp/$name/rs/moq-mux" && GLIBC_TUNABLES=$tunables CRITERION_HOME=/tmp/crit/$name/tun "$(cat /tmp/bin-$name)" --bench --warm-up-time 1 --measurement-time 2 --sample-size 30 --noplot "ts_schedule/$case_" --exact 2>&1 | grep -E "time:") + done + done + done + - name: Path length + run: | + set -uo pipefail + for n in 1 8 16 24 32 40 48 56; do + home=/tmp/h/$(printf 'x%.0s' $(seq 1 "$n")) + for name in base current; do + printf '%s len=%s ' "$name" "$n" + (cd "/tmp/$name/rs/moq-mux" && CRITERION_HOME=$home "$(cat /tmp/bin-$name)" --bench --warm-up-time 0.5 --measurement-time 1 --noplot ts_schedule/1/1 --exact 2>&1 | grep -E "time:") + done + done From 427b1791ffa5aa72fc4cbc50960e1371ba428a49 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 24 Oct 2026 23:13:13 -0700 Subject: [PATCH 3/4] ci: rerun the ts_schedule diagnostic the way Nightly runs it Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ts-schedule-diag.yml | 79 ++++++++++---------------- 1 file changed, 29 insertions(+), 50 deletions(-) diff --git a/.github/workflows/ts-schedule-diag.yml b/.github/workflows/ts-schedule-diag.yml index 6de16687132..f2323fce1df 100644 --- a/.github/workflows/ts-schedule-diag.yml +++ b/.github/workflows/ts-schedule-diag.yml @@ -9,65 +9,44 @@ permissions: jobs: diag: runs-on: ubuntu-24.04-arm - timeout-minutes: 50 + timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - name: Tools - run: sudo apt-get update -q && sudo apt-get install -y -q strace jq - - name: Build - run: | - set -euo pipefail - for side in base:4381af6f07e32d594036723f80d298b67e60085f current:0513c07369357ae0ecd6109d0f6afc61d16b597f; do - name=${side%%:*} rev=${side#*:} - git worktree add --detach "/tmp/$name" "$rev" - (cd "/tmp/$name" && CARGO_TARGET_DIR=/tmp/t-$name cargo bench --locked -p moq-mux --bench ts_schedule --no-run --message-format=json-render-diagnostics) | - jq -r 'select(.reason == "compiler-artifact" and .executable != null and .target.name == "ts_schedule") | .executable' >"/tmp/bin-$name" - cat "/tmp/bin-$name" - done - nproc; lscpu | head -20; getconf PAGESIZE; ldd --version | head -1 - - name: Timing - run: | - set -uo pipefail - for round in 1 2; do - for case_ in 1/1 4/1 16/1 1/64; do - for name in base current; do - printf '%s %s r%s ' "$name" "$case_" "$round" - (cd "/tmp/$name/rs/moq-mux" && CRITERION_HOME=/tmp/crit/$name/$round "$(cat /tmp/bin-$name)" --bench --warm-up-time 1 --measurement-time 2 --sample-size 30 --noplot "ts_schedule/$case_" --exact 2>&1 | grep -E "time:") - done - done - done - - name: Syscalls and faults + run: sudo apt-get update -q && sudo apt-get install -y -q strace + - uses: ./.github/actions/nix + - name: Rust cache + uses: ./.github/actions/rust-cache + with: + access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }} + secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }} + - name: Paired run as Nightly does it + env: + MOQ_BENCH_BASE: 4381af6f07e32d594036723f80d298b67e60085f + MOQ_BENCH_ARTIFACTS: /tmp/runs run: | - set -uo pipefail - for case_ in 1/1 4/1 16/1; do - for name in base current; do - echo "== $name $case_" - (cd "/tmp/$name/rs/moq-mux" && CRITERION_HOME=/tmp/crit/$name/sys strace -f -c -e trace=brk,mmap,munmap,madvise,mremap -o /tmp/strace.txt "$(cat /tmp/bin-$name)" --bench --profile-time 2 "ts_schedule/$case_" --exact >/dev/null 2>&1) - cat /tmp/strace.txt - (cd "/tmp/$name/rs/moq-mux" && CRITERION_HOME=/tmp/crit/$name/sys /usr/bin/time -f "faults minor=%R major=%F" "$(cat /tmp/bin-$name)" --bench --profile-time 2 "ts_schedule/$case_" --exact 2>&1 >/dev/null | grep faults) - done - done - - name: Allocator held fixed + nix develop --command bench/run.sh --criterion --no-fail-fast -p moq-relay -p moq-stats -p moq-mux -p moq-rtmp -p moq-uring -p moq-pattern \ + -p moq-quic --features moq-stats/bench,moq-quic/fuzz \ + --bench http_idle --bench auth --bench http_auth --bench producer_tick --bench decode --bench aggregate \ + --bench catalog --bench ts_schedule --bench export --bench containers --bench matching \ + --bench chunk_interleaving --bench worker_turns \ + --bench streams --bench receive --bench endpoint --bench wire -- ts_schedule/ 2>&1 | grep -E "Criterion|ts_schedule/|paired|regress|slower|faster|Change|change" || true + - name: Syscalls in the dev shell run: | set -uo pipefail - for tunables in glibc.malloc.trim_threshold=1073741824:glibc.malloc.mmap_threshold=33554432 glibc.malloc.mmap_threshold=131072; do + run=$(ls -d /tmp/runs/run.* | head -1) + for side in base current; do + line=$(grep -P '^moq-mux/ts_schedule\t' "$run/criterion-$side.targets") + package=$(cut -f2 <<<"$line") bin=$(cut -f3 <<<"$line") + echo "== $side $bin" + ldd "$bin" | grep libc for case_ in 1/1 4/1 16/1; do - for name in base current; do - printf '%s %s %s ' "$tunables" "$name" "$case_" - (cd "/tmp/$name/rs/moq-mux" && GLIBC_TUNABLES=$tunables CRITERION_HOME=/tmp/crit/$name/tun "$(cat /tmp/bin-$name)" --bench --warm-up-time 1 --measurement-time 2 --sample-size 30 --noplot "ts_schedule/$case_" --exact 2>&1 | grep -E "time:") - done - done - done - - name: Path length - run: | - set -uo pipefail - for n in 1 8 16 24 32 40 48 56; do - home=/tmp/h/$(printf 'x%.0s' $(seq 1 "$n")) - for name in base current; do - printf '%s len=%s ' "$name" "$n" - (cd "/tmp/$name/rs/moq-mux" && CRITERION_HOME=$home "$(cat /tmp/bin-$name)" --bench --warm-up-time 0.5 --measurement-time 1 --noplot ts_schedule/1/1 --exact 2>&1 | grep -E "time:") + (cd "$package" && CRITERION_HOME=/tmp/sys/$side /usr/bin/strace -c -e trace=brk,mmap,munmap,madvise,mremap -o /tmp/strace.txt "$bin" --bench --profile-time 2 "ts_schedule/$case_" --exact >/dev/null 2>&1) + echo "-- $case_"; cat /tmp/strace.txt + (cd "$package" && CRITERION_HOME=/tmp/sys/$side "$bin" --bench --warm-up-time 1 --measurement-time 2 --sample-size 30 --noplot "ts_schedule/$case_" --exact 2>&1 | grep time:) done done + env | grep -iE "malloc|tunable" || echo "no malloc env" From ae16f42f7301d3075da577298aae5d5a7f884fcc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 25 Oct 2026 02:02:02 -0700 Subject: [PATCH 4/4] fix(mux): pin glibc's heap in the ts_schedule bench A padded slot at the bench's 100 Mb/s lays out about 300 KB, where glibc adapts between mmap and the heap and trims the heap's top. Which mode a process settles on depends on its heap history, not the schedule: on the ARM runners Nightly's build of an unchanged schedule faulted fresh pages every slot, reading 5x slower than the previous night. The bench now holds both thresholds so every slot reuses the heap. Drop the temporary ARM diagnostic workflow and the finished quest. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ts-schedule-diag.yml | 52 -------------------------- Cargo.lock | 1 + quest/m0/nightly/README.md | 1 - quest/m0/nightly/ts-schedule.md | 21 ----------- rs/moq-mux/Cargo.toml | 4 ++ rs/moq-mux/benches/ts_schedule.rs | 16 ++++++++ 6 files changed, 21 insertions(+), 74 deletions(-) delete mode 100644 .github/workflows/ts-schedule-diag.yml delete mode 100644 quest/m0/nightly/ts-schedule.md diff --git a/.github/workflows/ts-schedule-diag.yml b/.github/workflows/ts-schedule-diag.yml deleted file mode 100644 index f2323fce1df..00000000000 --- a/.github/workflows/ts-schedule-diag.yml +++ /dev/null @@ -1,52 +0,0 @@ -# Temporary diagnostic for quest/m0/nightly/ts-schedule.md; removed before the PR leaves draft. -name: ts-schedule-diag -on: - push: - branches: [quest/m0/nightly/ts-schedule] - paths: [.github/workflows/ts-schedule-diag.yml] -permissions: - contents: read -jobs: - diag: - runs-on: ubuntu-24.04-arm - timeout-minutes: 60 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - name: Tools - run: sudo apt-get update -q && sudo apt-get install -y -q strace - - uses: ./.github/actions/nix - - name: Rust cache - uses: ./.github/actions/rust-cache - with: - access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }} - secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }} - - name: Paired run as Nightly does it - env: - MOQ_BENCH_BASE: 4381af6f07e32d594036723f80d298b67e60085f - MOQ_BENCH_ARTIFACTS: /tmp/runs - run: | - nix develop --command bench/run.sh --criterion --no-fail-fast -p moq-relay -p moq-stats -p moq-mux -p moq-rtmp -p moq-uring -p moq-pattern \ - -p moq-quic --features moq-stats/bench,moq-quic/fuzz \ - --bench http_idle --bench auth --bench http_auth --bench producer_tick --bench decode --bench aggregate \ - --bench catalog --bench ts_schedule --bench export --bench containers --bench matching \ - --bench chunk_interleaving --bench worker_turns \ - --bench streams --bench receive --bench endpoint --bench wire -- ts_schedule/ 2>&1 | grep -E "Criterion|ts_schedule/|paired|regress|slower|faster|Change|change" || true - - name: Syscalls in the dev shell - run: | - set -uo pipefail - run=$(ls -d /tmp/runs/run.* | head -1) - for side in base current; do - line=$(grep -P '^moq-mux/ts_schedule\t' "$run/criterion-$side.targets") - package=$(cut -f2 <<<"$line") bin=$(cut -f3 <<<"$line") - echo "== $side $bin" - ldd "$bin" | grep libc - for case_ in 1/1 4/1 16/1; do - (cd "$package" && CRITERION_HOME=/tmp/sys/$side /usr/bin/strace -c -e trace=brk,mmap,munmap,madvise,mremap -o /tmp/strace.txt "$bin" --bench --profile-time 2 "ts_schedule/$case_" --exact >/dev/null 2>&1) - echo "-- $case_"; cat /tmp/strace.txt - (cd "$package" && CRITERION_HOME=/tmp/sys/$side "$bin" --bench --warm-up-time 1 --measurement-time 2 --sample-size 30 --noplot "ts_schedule/$case_" --exact 2>&1 | grep time:) - done - done - env | grep -iE "malloc|tunable" || echo "no malloc env" diff --git a/Cargo.lock b/Cargo.lock index ee815167cb4..d20058e490d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5010,6 +5010,7 @@ dependencies = [ "h264-parser", "hang", "kio 0.6.2", + "libc", "memchr", "moq-clock", "moq-flate", diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index 39fd3c3db0b..db5409094a1 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -36,6 +36,5 @@ Public API: none. Wire: none. - [Claimed fetches wait for credit](/quest/m0/nightly/drill-credit.md) - the impaired burst drills stop losing groups to "no stream credit" - [Feature shapes in the gate](/quest/m0/nightly/feature-shapes.md) - feature-gated dead code fails the PR that adds it, not the next Nightly - [qmux fallback closes cleanly](/quest/m0/nightly/qmux-fallback-close.md) - the WebSocket fallback startup benchmark stops failing "not cleanly closed" -- [ts_schedule regression](/quest/m0/nightly/ts-schedule.md) - the moq-mux `ts_schedule` benches return to their old cost - [Nightly's base in a shallow checkout](/quest/m0/nightly/shallow-base.md) - `sh/gh/nightly.test.py` passes in Nightly's depth-1 checkout - [Advisory bumps](/quest/m0/nightly/advisories.md) - both audits pass on today's advisories diff --git a/quest/m0/nightly/ts-schedule.md b/quest/m0/nightly/ts-schedule.md deleted file mode 100644 index 37eb7aeec30..00000000000 --- a/quest/m0/nightly/ts-schedule.md +++ /dev/null @@ -1,21 +0,0 @@ -# [S] ts_schedule regression - -## Goal - -`rust/shared/ts_schedule/{1/1, 1/16, 4/1}` (`rs/moq-mux/benches/ts_schedule.rs`) -return to their cost before the regression, and Nightly's paired gate on -the shared bench targets passes. - -## Plan - -Nightly's first paired run (0513c07369 against 4381af6f0) measured them -400 to 460% slower, too large for noise. The window holds -[moq-dev/moq#6417](https://github.com/moq-dev/moq/pull/6417), -[moq-dev/moq#6533](https://github.com/moq-dev/moq/pull/6533), and -[moq-dev/moq#6527](https://github.com/moq-dev/moq/pull/6527). Bisect, then -fix the cost at its source, or show the bench now measures more work and -rebaseline with that reason. The smaller `server_admission` (+20%) and -`uring-session_lite/no-gso` (+11%) deltas belong to the threshold -calibration in [bench CI](/quest/m1/bench-ci.md). - -Public API: none. Wire: none. diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index ae1d50866fc..636187bb92a 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -49,6 +49,10 @@ moq-clock = { workspace = true, features = ["sim"] } tokio = { workspace = true, features = ["test-util"] } tracing-test = { workspace = true, features = ["no-env-filter"] } +# The ts_schedule bench pins glibc's heap. +[target.'cfg(all(target_os = "linux", target_env = "gnu"))'.dev-dependencies] +libc = { workspace = true } + [[bench]] name = "catalog" harness = false diff --git a/rs/moq-mux/benches/ts_schedule.rs b/rs/moq-mux/benches/ts_schedule.rs index 40cf20bc333..89be607de95 100644 --- a/rs/moq-mux/benches/ts_schedule.rs +++ b/rs/moq-mux/benches/ts_schedule.rs @@ -22,7 +22,22 @@ fn packet(pid: u16) -> [u8; 188] { packet } +/// Pin glibc's heap so every slot reuses it. A padded slot at 100 Mb/s lays out about 300 KB, +/// right at the size where glibc adapts between `mmap` and the heap and trims the heap's +/// top, so which one a run settles on, and with it fresh page faults on every slot, flips +/// with the process's heap history alone: that was a 5x swing on the ARM runners with no +/// change to the schedule. +fn hold_heap() { + #[cfg(all(target_os = "linux", target_env = "gnu"))] + // SAFETY: mallopt only tunes the allocator, called before anything is measured. + unsafe { + assert_eq!(libc::mallopt(libc::M_MMAP_THRESHOLD, 32 << 20), 1); + assert_eq!(libc::mallopt(libc::M_TRIM_THRESHOLD, libc::c_int::MAX), 1); + } +} + fn admission(c: &mut Criterion) { + hold_heap(); let mut group = c.benchmark_group("ts_schedule"); group.measurement_time(Duration::from_secs(1)); group.warm_up_time(Duration::from_millis(200)); @@ -76,6 +91,7 @@ fn admission(c: &mut Criterion) { /// Padded layout across queued tracks and access-unit size, each track's first unit with the /// tables ahead of it, the larger units spanning several slots. fn tables(c: &mut Criterion) { + hold_heap(); let mut group = c.benchmark_group("ts_schedule_tables"); group.measurement_time(Duration::from_secs(1)); group.warm_up_time(Duration::from_millis(200));