From 4759c3832fab75699020ea2ff9290f9e5ba09a57 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:21:10 -0700 Subject: [PATCH 1/2] quest: claim quest/m0/ci/nightly-rust-bench From 546dd5f0cfd010fc88a7c662929e3af5b1af716d Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:26:47 -0700 Subject: [PATCH 2/2] ci(nightly): split the Rust benchmarks into two legs The Rust benchmark group was cancelled at its 60-minute limit on 2026-10-16 before moq-gst's catalog bench and moq-cli's stream import ran. Split it by harness so each gets its own hour: nextest's ignored benchmark tests (release builds, then test-profile ones) and cargo's bench targets. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 13 ++++++----- quest/m0/ci/README.md | 1 - quest/m0/ci/nightly-green.md | 1 - quest/m0/ci/nightly-rust-bench.md | 37 ------------------------------- quest/m2/bench-site.md | 6 ++--- sh/gh/nightly.sh | 24 +++++++++++--------- sh/gh/nightly.test.py | 21 +++++++++++++----- 7 files changed, 40 insertions(+), 63 deletions(-) delete mode 100644 quest/m0/ci/nightly-rust-bench.md diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index db74fda80c6..4c54c2bb677 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -46,9 +46,10 @@ jobs: x11: uses: ./.github/workflows/x11.yml - # Each independent workload gets its own hour. Rust and JS benchmark groups - # keep one build environment each, so splitting the feature compile does not - # turn every benchmark into a fresh compile of the same dependencies. + # Each independent workload gets its own hour. Each benchmark group keeps one + # build environment, so a benchmark reuses the dependencies its group already + # compiled. The Rust benchmarks outgrew one hour, so they split by harness: + # nextest's ignored benchmark tests and cargo's bench targets share few builds. nightly: name: Nightly (${{ matrix.name }}) runs-on: ubuntu-24.04-arm @@ -74,8 +75,10 @@ jobs: - name: Relay lock-wait profile recipe: bench-lock fanout-heavy 4 perf: true - - name: Rust benchmarks - recipe: gh nightly rust + - name: Rust benchmark tests + recipe: gh nightly rust-test + - name: Rust bench targets + recipe: gh nightly rust-bench - name: JS benchmarks recipe: gh nightly js uncached: true diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index 44eaf6d2a7b..28744c3de71 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -68,6 +68,5 @@ The historical samples locate candidates; they are not controlled benchmarks. - [Require platform gates](/quest/m0/ci/platform-queue.md) - activate the consolidated platform checks in the main and release rulesets after the workflow lands - [mbx on R2](/quest/m0/ci/mbx-r2.md) - Rust builds on every OS read an R2 cache that only `main` writes, escaping the 10 GB Actions cache - [Nix cache](/quest/m0/ci/nix-cache.md) - `nix.moq.dev` on R2 replaces Cachix for CI and releases -- [Rust benchmark group fits its hour](/quest/m0/ci/nightly-rust-bench.md) - Nightly's Rust benchmark workload finishes inside its job limit - [Nightly workloads pass](/quest/m0/ci/nightly-green.md) - the Nightly failures found on 2026-10-16 are fixed - [First shared FFI release](/quest/m0/ci/nightly-ffi-reuse.md) - condition: the first `moq-ffi-v*` tag that runs `Release FFI`, and a `Shipping sizes` run; verify the wrappers and size keys diff --git a/quest/m0/ci/nightly-green.md b/quest/m0/ci/nightly-green.md index dcce9e2b979..d303dc8dc74 100644 --- a/quest/m0/ci/nightly-green.md +++ b/quest/m0/ci/nightly-green.md @@ -39,5 +39,4 @@ have not run yet; triage them from the next Nightly too. ## Related -- [Rust benchmark group fits its hour](/quest/m0/ci/nightly-rust-bench.md) - the cancelled Rust group - [First shared FFI release](/quest/m0/ci/nightly-ffi-reuse.md) - the check that found these diff --git a/quest/m0/ci/nightly-rust-bench.md b/quest/m0/ci/nightly-rust-bench.md deleted file mode 100644 index a9b3b43bb39..00000000000 --- a/quest/m0/ci/nightly-rust-bench.md +++ /dev/null @@ -1,37 +0,0 @@ -# [S] Rust benchmark group fits its hour - -## Goal - -Nightly's `Rust benchmarks` workload (`just gh nightly rust`) finishes inside -its 60-minute job limit with room to grow, so every Rust benchmark in -`sh/gh/nightly.sh` runs each night. - -## Plan - -On 2026-10-16 (run 38897397428) it was cancelled at 60 minutes, partway -through `bench-gst-catalog`, so `moq-cli`'s stream-import benchmark never ran. -The hour split: - -- 13.5 minutes entering the dev shell: `uniffi-bindgen-go`, - `uniffi-bindgen-cpp` and `quest` built from source because the shell - closure wasn't in the binary cache. The five other Nightly workload logs - checked show the same 12-14 minutes. -- 46 minutes of benchmarks, 34 of them compiling: the group mixes - `test`, `release` and `bench` profiles and several feature sets, so shared - crates compile more than once. The largest steps were the multi-crate - `cargo bench` (8 minutes compiling) and `bench-feed-fetch` (6 minutes). - -Benchmarks are still being added to the group (#5663, #5675, #5698 since that -run), so caching the shell alone leaves it close to the limit. - -Decided: split the group into two matrix legs by build profile (release -benches/tests vs the `test`-profile ignored tests), each with its own hour. -That bounds each leg and keeps the shared-build rationale within each leg. -Raising `timeout-minutes` alone lets the group grow into whatever limit it -gets, and [the Nix cache](/quest/m0/ci/nix-cache.md) recovers the 13.5 -minutes but isn't enough by itself. - -## Related - -- [nix.moq.dev replaces Cachix](/quest/m0/ci/nix-cache.md) - caches the dev shell every Nightly job builds -- [Nightly workloads pass](/quest/m0/ci/nightly-green.md) - the other Nightly failures from the same run diff --git a/quest/m2/bench-site.md b/quest/m2/bench-site.md index 4d9e56f7228..d7b3e2608bf 100644 --- a/quest/m2/bench-site.md +++ b/quest/m2/bench-site.md @@ -15,9 +15,9 @@ Decided with the maintainer on 2026-10-15: - **GitHub Pages via github-action-benchmark**: the nightly pushes each run's results to a `gh-pages` branch, and the action's built-in charts are the site. Its own alerting stays off; tracking is informational. -- **Sources**: the binding benchmark's JSON and the "Rust benchmarks" and - "JS benchmarks" nightly jobs, converted to the action's input format where - they don't already emit one. Several are smoke runs today: most Rust +- **Sources**: the binding benchmark's JSON and the "Rust benchmark tests", + "Rust bench targets" and "JS benchmarks" nightly jobs, converted to the + action's input format where they don't already emit one. Several are smoke runs today: most Rust Criterion targets run with `-- --test`, which measures nothing. Switch the charted targets to a measuring mode (or add a measurement pass), weighing the added nightly runtime, and check each JS bench reports a measurement diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 43f3bb46592..2e42f51e400 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -2,7 +2,7 @@ # Keep benchmark builds together while unrelated Nightly workloads run independently. set -euo pipefail -usage='usage: just gh nightly rust|js' +usage='usage: just gh nightly rust-test|rust-bench|js' if [[ $# != 1 ]]; then echo "$usage" >&2 exit 2 @@ -15,7 +15,8 @@ run() { } case "${1:-}" in - rust) + # Ignored benchmark tests under nextest: the release builds first, then the test-profile ones. + rust-test) run just rs bench-latency run just rs test -p moq-bench --release --run-ignored only \ -E 'test(benchmark_readiness_establishment)' --success-output immediate @@ -23,8 +24,18 @@ case "${1:-}" in run just rs test --release -p moq-quic -E 'test(benchmark_ack_cleanup)' \ --run-ignored only --success-output immediate run just rs bench-feed-fetch - run just rs test -p moq-relay -E 'test(benchmark_stun_sources)' --run-ignored only --success-output immediate run just rs test -p moq-quic --release -E 'test(benchmark_receive_credit)' --run-ignored only --success-output immediate + run just rs test -p moq-relay -E 'test(benchmark_stun_sources)' --run-ignored only --success-output immediate + run just rs test -p moq-archive -p moq-mux -p moq-hls --no-fail-fast \ + -E 'test(benchmark_timeline_fetch_fanout) | test(benchmark_damage_pid_fanout) | test(benchmark_pes_staging) | test(benchmark_reference_handoff_fanout)' \ + --run-ignored only --success-output immediate + # Codecs are irrelevant to timebase reservation, so skip building them. + run just rs test -p moq-ffi --no-default-features -E 'test(benchmark_timebase_reservation_fanout)' \ + --run-ignored only --success-output immediate + run just rs test -p moq-cli -E 'test(benchmark_stream_import)' --run-ignored only --success-output immediate + ;; + # Cargo bench targets, all in the bench profile. + rust-bench) # Fuzz exposes moq-net's codec benches; it is not a feature of the other crates. run cargo bench --locked --no-fail-fast -p moq-net --features fuzz --bench '*' -- --test # Exercise the optional thread-CPU clock as well as the normal elapsed-time smoke. @@ -39,15 +50,8 @@ case "${1:-}" in --bench catalog --bench ts_schedule --bench export --bench containers --bench matching \ --bench chunk_interleaving --bench worker_turns \ --bench streams --bench receive --bench endpoint --bench wire -- --test - run just rs test -p moq-archive -p moq-mux -p moq-hls --no-fail-fast \ - -E 'test(benchmark_timeline_fetch_fanout) | test(benchmark_damage_pid_fanout) | test(benchmark_pes_staging) | test(benchmark_reference_handoff_fanout)' \ - --run-ignored only --success-output immediate - # Codecs are irrelevant to timebase reservation, so skip building them. - run just rs test -p moq-ffi --no-default-features -E 'test(benchmark_timebase_reservation_fanout)' \ - --run-ignored only --success-output immediate # moq-gst's catalog bench shares a target name with moq-mux's, so it runs on its own. run just rs bench-gst-catalog -- --quick - run just rs test -p moq-cli -E 'test(benchmark_stream_import)' --run-ignored only --success-output immediate ;; js) for benchmark in broadcasts request-ids requests forward reader frames varint lite-varint track interests priority; do diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index 2fbcb8eae6f..b90275de488 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -36,8 +36,8 @@ def test_failed_js_benchmark_does_not_hide_later_work(self): self.assertEqual(calls[-1], "just js signals-bench") self.assertIn("failed: just js bench reader", result.stderr) - def test_release_benchmarks_share_one_invocation(self): - result, calls = self.run_group("rust") + def test_bench_targets_share_one_invocation(self): + result, calls = self.run_group("rust-bench") self.assertEqual(result.returncode, 0, result.stderr) release = [call for call in calls if call.startswith("cargo bench")] self.assertEqual(len(release), 4) # moq-net, its CPU clock, kio, then the shared build. @@ -53,16 +53,25 @@ def test_release_benchmarks_share_one_invocation(self): self.assertIn("--bench containers", shared[0]) self.assertIn("-p moq-quic", shared[0]) self.assertIn("--bench wire", shared[0]) - self.assertIn("just rs bench-gst-catalog -- --quick", calls) + self.assertEqual(calls[-1], "just rs bench-gst-catalog -- --quick") + + def test_benchmark_tests_stay_out_of_cargo_bench(self): + result, calls = self.run_group("rust-test") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(any(call.startswith("cargo bench") for call in calls)) + # Release builds run before the test-profile ones. + profiles = ["--release" in call or " bench-" in call for call in calls] + self.assertEqual(profiles, sorted(profiles, reverse=True)) + self.assertIn("just rs bench-feed-fetch", calls) self.assertTrue(calls[-1].startswith("just rs test -p moq-cli")) def test_failed_release_build_does_not_hide_later_work(self): - _, calls = self.run_group("rust") + _, calls = self.run_group("rust-bench") build = next(call.removeprefix("cargo ") for call in calls if call.startswith("cargo bench") and "-p moq-relay" in call) - result, calls = self.run_group("rust", build) + result, calls = self.run_group("rust-bench", build) self.assertEqual(result.returncode, 1) - self.assertTrue(calls[-1].startswith("just rs test -p moq-cli")) + self.assertEqual(calls[-1], "just rs bench-gst-catalog -- --quick") def test_unknown_group_fails_before_running_anything(self): result, calls = self.run_group("unknown")