From 6cf3322772ff4b8d108cd9e82c7a3ea52e4c39d9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 02:18:44 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m0/nightly/bench-timeouts.md Quest-Claim: 621da10e651c1526 From 583ea45637e7a39f5d443af03ab7794391aaf984 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 03:19:27 -0800 Subject: [PATCH 2/5] wip: nightly bench timeouts --- .github/workflows/nightly.yml | 15 +++++++++ bench/run.sh | 58 +++++++++++++++++++++++++++++------ js/crypto/bench/pump.ts | 2 ++ js/json/bench/stream.ts | 9 ++++-- js/net/bench/requests.ts | 15 ++++----- sh/gh/nightly.sh | 24 +++++++++++++-- sh/gh/previous-nightly.sh | 18 ++++++++--- 7 files changed, 113 insertions(+), 28 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f4a337f0bff..be855ee7c8a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -251,12 +251,17 @@ jobs: access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }} secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }} + # Inside the job's hour, so a workload that overruns fails the job, where the + # failure-based triage sees it, instead of cancelling it, and later steps still run. - name: Run workload id: workload + timeout-minutes: 55 run: nix develop ${{ matrix.shell }} --command just ${{ matrix.recipe }} env: NEXTEST_PROFILE: default BENCH_BASE: ${{ matrix.paired && (inputs.bench-base || 'previous-nightly') || '' }} + # The previous Nightly's copy is the paired base's executables (see sh/gh/nightly.sh). + BENCH_BUILDS_ARTIFACT: ${{ matrix.paired && format('builds-{0}', matrix.artifact) || '' }} GH_TOKEN: ${{ matrix.paired && github.token || '' }} - name: Record benchmark outcome @@ -271,6 +276,16 @@ jobs: path: ${{ matrix.artifact-path }} retention-days: 45 + # Only scheduled runs serve as a later run's base. + - name: Keep bench executables + if: ${{ matrix.paired && github.event_name == 'schedule' && !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: builds-${{ matrix.artifact }} + path: target/bench/builds + if-no-files-found: ignore + retention-days: 3 + # After the upload, so the benchmark site still records tonight's history and # publishes the comparison that failed this job before the alert links to it. - name: Fail on paired regressions diff --git a/bench/run.sh b/bench/run.sh index d4c988cc024..e4e3f9dc58a 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -496,6 +496,38 @@ unavailable() { printf 'Paired comparison unavailable for %s: %s\n' "$1" "$2" >&2 } +# Copy the executables a list names to BENCH_KEEP_BUILDS, with the commit they were built +# from, so a later run can measure them as its base without relinking them. +keep_builds() { + local list=$1 dir=$BENCH_KEEP_BUILDS index=0 key package executable + rm -rf "$dir" + mkdir -p "$dir/bin" + (cd "$ROOT" && git rev-parse HEAD) >"$dir/commit" + : >"$dir/targets.tsv" + while IFS=$'\t' read -r key package executable; do + cp "$executable" "$dir/bin/$index" + printf '%s\t%s\t%s\n' "$key" "${package#"$ROOT"/}" "bin/$index" >>"$dir/targets.tsv" + index=$((index + 1)) + done <"$list" +} + +# List the base's kept executables in OUT, as criterion_build would, from the base checkout's +# packages. Fails when BENCH_BASE_BUILDS holds no builds of BASE covering every current target. +kept_base() { + local out=$1 current=$2 dir=${BENCH_BASE_BUILDS:-} key package file + [[ -n $dir && -s $dir/targets.tsv && -f $dir/commit && $(<"$dir/commit") == "$BASE" ]] || return 1 + dir=$(cd "$dir" && pwd -P) + while IFS=$'\t' read -r key _; do + [[ -n $(target_line "$dir/targets.tsv" "$key") ]] || return 1 + done <"$current" + : >"$out" + while IFS=$'\t' read -r key package file; do + # An artifact download drops the executable bit. + chmod +x "$dir/$file" + printf '%s\t%s\t%s\n' "$key" "$WORKTREE/$package" "$dir/$file" >>"$out" + done <"$dir/targets.tsv" +} + # Measure the selected targets in quick mode. With a base, run each target's base build # beside it, alternating which goes first, then confirm the screen's worst slowdowns. # A failed current target fails the run; the base only marks its targets unavailable. @@ -506,6 +538,7 @@ measure_criterion() { local key side targets index=0 status=0 local filters=(${CRITERION_FILTER_ARGS[@]+"${CRITERION_FILTER_ARGS[@]}"}) criterion_build "$current" "$ROOT" ${CRITERION_CARGO_ARGS[@]+"${CRITERION_CARGO_ARGS[@]}"} + if [[ -n ${BENCH_KEEP_BUILDS:-} ]]; then keep_builds "$current"; fi mapfile -t targets < <(cut -f1 "$current") if ((${#targets[@]} == 0)); then printf 'no Criterion bench targets selected\n' >&2 @@ -517,15 +550,22 @@ measure_criterion() { mkdir -p "$RUN/base-context" (cd "$WORKTREE" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context" "$BASE" comparison-base) printf 'Criterion paired with base %s\n' "$BASE" - printf 'Base build log: %s/base-build.log\n' "$RUN" - local build_status=(0 0) - criterion_build_base "$base" 2>&1 | tee "$RUN/base-build.log" >&2 || build_status=("${PIPESTATUS[@]}") - if ((build_status[1] != 0)); then - printf 'Failed to write base build log: %s/base-build.log\n' "$RUN" >&2 - fi - if ((build_status[0] != 0)); then - : >"$base" - for key in "${targets[@]}"; do unavailable "$key" "base build failed"; done + if kept_base "$base" "$current"; then + printf 'Base executables kept by an earlier run: %s\n' "$BENCH_BASE_BUILDS" + else + if [[ -n ${BENCH_BASE_BUILDS:-} ]]; then + printf 'No kept base executable of every target in %s; building the base\n' "$BENCH_BASE_BUILDS" + fi + printf 'Base build log: %s/base-build.log\n' "$RUN" + local build_status=(0 0) + criterion_build_base "$base" 2>&1 | tee "$RUN/base-build.log" >&2 || build_status=("${PIPESTATUS[@]}") + if ((build_status[1] != 0)); then + printf 'Failed to write base build log: %s/base-build.log\n' "$RUN" >&2 + fi + if ((build_status[0] != 0)); then + : >"$base" + for key in "${targets[@]}"; do unavailable "$key" "base build failed"; done + fi fi fi : >"$RUN/paired-identical.txt" diff --git a/js/crypto/bench/pump.ts b/js/crypto/bench/pump.ts index 9d04ecf1921..abf3c792b59 100644 --- a/js/crypto/bench/pump.ts +++ b/js/crypto/bench/pump.ts @@ -24,8 +24,10 @@ for (const bytes of [160, 1_160, 16_384, 262_144]) { // The first round warms the JIT and key imports and is discarded. for (let round = 0; round < 6; round++) { const generation = credential.produce(); + // No publisher limit: every group is sealed before the first is read. const net = new Broadcast.Producer().createTrack(await generation.name("video"), { timescale: Time.Timescale.MILLI, + maxAge: null, }); const consumer = credential .consume(generation.epoch) diff --git a/js/json/bench/stream.ts b/js/json/bench/stream.ts index b1d6b9bb2a3..a2bdad4ed99 100644 --- a/js/json/bench/stream.ts +++ b/js/json/bench/stream.ts @@ -1,7 +1,10 @@ /** Sweep buffered JSON stream reads over record count, payload size, and track compression. */ -import { Track } from "@moq/net"; +import { Time, Track } from "@moq/net"; import { Consumer, Producer } from "../src/stream/index.ts"; +// A scale-less log opens its one group at this key, and every record carries it. +const at = Time.Timestamp.unscaled(0n); + console.log("records,payload_bytes,compress,ns_per_record"); for (const records of [64, 1024]) { for (const bytes of [16, 1024]) { @@ -11,14 +14,14 @@ for (const records of [64, 1024]) { const track = new Track.Producer("stream-bench").accept({ compression: compress ? "deflate" : "none" }); const producer = new Producer<{ n: number; text: string }>({ track }); const text = "x".repeat(bytes); - for (let n = 0; n < records; n++) producer.append({ value: { n, text } }); + for (let n = 0; n < records; n++) producer.append({ value: { n, text }, at }); producer.finish(); const consumer = new Consumer<{ n: number; text: string }>({ track: track.subscribe() }); const start = performance.now(); for (let n = 0; n < records; n++) { const record = await consumer.next(); - if (record?.value.n !== n || record.value.text !== text || record.at !== undefined) { + if (record?.value.n !== n || record.value.text !== text || record.at.isBefore(at) || at.isBefore(record.at)) { throw new Error("stream benchmark lost a record"); } } diff --git a/js/net/bench/requests.ts b/js/net/bench/requests.ts index 49fd84a85cd..211a37c2460 100644 --- a/js/net/bench/requests.ts +++ b/js/net/bench/requests.ts @@ -27,11 +27,7 @@ for (const touched of ["watched", "unrelated"]) for (const followers of [1, 8, 32]) { const origin = new Producer(); const path = Path.from("room/0"); - const broadcasts = Array.from({ length: routes }, (_, i) => { - const broadcastRoute = origin.route(Path.from(`room/${i}`)); - const broadcast = broadcastRoute.broadcast; - return broadcast; - }); + const broadcasts = Array.from({ length: routes }, (_, i) => origin.route(Path.from(`room/${i}`))); const requests = Array.from({ length: followers }, () => origin.follow(path)); await settle(); let current = broadcasts[0]; @@ -40,8 +36,7 @@ for (const touched of ["watched", "unrelated"]) for (let round = 0; round < 6; round++) { const start = performance.now(); for (let i = 0; i < 128; i++) { - const nextRoute = origin.route(touched === "watched" ? path : Path.from("other")); - const next = nextRoute.broadcast; + const next = origin.route(touched === "watched" ? path : Path.from("other")); if (touched === "watched") { current.close(); current = next; @@ -50,8 +45,9 @@ for (const touched of ["watched", "unrelated"]) unrelated = next; } await settle(); + // A follower reads its own consumer of the route's broadcast, so match the instance. for (const request of requests) - if (request.active.peek()?.closed !== current.closed) + if (request.active.peek()?.instance !== current.instance) throw new Error("wrong publisher instance"); } if (round > 0) samples.push(((performance.now() - start) * 1000) / 128); @@ -142,7 +138,8 @@ for (const failovers of [0, 1, 16, 128]) const primary = origin.claim(prefix, { instance: { id: instance, epoch: 0n }, cost: 0n }); const ar = primary.requested(); const br = origin.claim(prefix, { instance: { id: instance, epoch: 0n }, cost: 1n }).requested(); - const held = origin.request(path); + // Pinned, or the path's earlier instance would answer it and this claim never hears it. + const held = origin.request(path, { instance }); const source = new Broadcast(); sources.push(source); const answer = (await ar.next()).value; diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index 9dc255c6d43..a19264d2d4b 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -22,13 +22,25 @@ esac # BENCH_BASE pairs every Rust bench target with its build at that revision, run on the # same machine; `previous-nightly` is the commit of the latest other scheduled Nightly on main. +# That run kept its bench executables as the BENCH_BUILDS_ARTIFACT artifact. The cache +# can't hold a linked executable, so reusing them saves relinking every target at the base. +base_builds= if [[ -n ${BENCH_BASE:-} ]]; then if [[ $report != rust ]]; then echo "nightly: BENCH_BASE applies only to the Rust bench groups" >&2 exit 2 fi if [[ $BENCH_BASE == previous-nightly ]]; then - BENCH_BASE=$(bash "$(dirname "${BASH_SOURCE[0]}")/previous-nightly.sh") + read -r base_run BENCH_BASE < <(bash "$(dirname "${BASH_SOURCE[0]}")/previous-nightly.sh" --run) || exit 1 + if [[ -n ${BENCH_BUILDS_ARTIFACT:-} ]]; then + base_builds=$(git rev-parse --show-toplevel)/target/bench/base-builds/$1 + rm -rf "$base_builds" + # A previous leg that ended before its builds kept none; the base is built instead. + if ! gh run download "$base_run" --name "$BENCH_BUILDS_ARTIFACT" --dir "$base_builds"; then + echo "nightly: run $base_run kept no $BENCH_BUILDS_ARTIFACT; building the base" >&2 + base_builds= + fi + fi fi # A shallow CI checkout has only its own commit. Fetching a branch name only updates # FETCH_HEAD, so the base is exported as the commit it resolved to. @@ -39,9 +51,12 @@ if [[ -n ${BENCH_BASE:-} ]]; then export BENCH_BASE=$base fi +group=$1 if [[ -n $report ]]; then root=$(git rev-parse --show-toplevel) out=${BENCH_RESULTS:-$root/target/bench/nightly/$1} + builds=$root/target/bench/builds/$group + rm -rf "$builds" mkdir -p "$out" export BENCH_RUNS="$out/criterion-runs.txt" : >"$BENCH_RUNS" @@ -52,10 +67,15 @@ fi # shellcheck source=sh/checks.sh source "$(dirname "${BASH_SOURCE[0]}")/../checks.sh" +# Each label keeps its executables for the next Nightly's base, under the directory +# the workflow uploads. criterion() { local label=$1 shift - run env BENCH_LABEL="$label" BENCH_ARTIFACTS="$out/runs" just gh criterion "$@" + run env BENCH_LABEL="$label" BENCH_ARTIFACTS="$out/runs" \ + BENCH_KEEP_BUILDS="$builds/$label" \ + BENCH_BASE_BUILDS="${base_builds:+$base_builds/$group/$label}" \ + just gh criterion "$@" } javascript() { diff --git a/sh/gh/previous-nightly.sh b/sh/gh/previous-nightly.sh index 35a502a3874..829270c47b0 100644 --- a/sh/gh/previous-nightly.sh +++ b/sh/gh/previous-nightly.sh @@ -1,11 +1,19 @@ #!/usr/bin/env bash -# Print the commit of the latest scheduled Nightly on main other than this run. +# Print the commit of the latest scheduled Nightly on main other than this run; +# with --run, its run ID and then the commit. set -euo pipefail -sha=$(gh run list --workflow nightly.yml --branch main --event schedule --limit 2 \ - --json databaseId,headSha --jq "map(select(.databaseId != ${GITHUB_RUN_ID:-0}))[0].headSha // empty") -if [[ -z $sha ]]; then +fields='.headSha' +if [[ ${1:-} == --run ]]; then + fields='"\(.databaseId) \(.headSha)"' +elif [[ $# != 0 ]]; then + echo "usage: previous-nightly.sh [--run]" >&2 + exit 2 +fi +line=$(gh run list --workflow nightly.yml --branch main --event schedule --limit 2 \ + --json databaseId,headSha --jq "map(select(.databaseId != ${GITHUB_RUN_ID:-0}))[0] // empty | $fields") +if [[ -z $line ]]; then echo "nightly: no previous scheduled Nightly on main" >&2 exit 1 fi -echo "$sha" +echo "$line" From b21a2892c742cc3108a9a2b358b2934494515002 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 04:02:47 -0800 Subject: [PATCH 3/5] ci(nightly): bench legs fit their hour: reuse the base's executables, fail overruns, fix hung JS benches Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 10 ++++-- js/json/bench/stream.ts | 7 +++- quest/m0/nightly/README.md | 4 ++- quest/m0/nightly/bench-timeouts.md | 24 -------------- sh/gh/nightly.test.py | 53 ++++++++++++++++++++++++++++-- 5 files changed, 67 insertions(+), 31 deletions(-) delete mode 100644 quest/m0/nightly/bench-timeouts.md diff --git a/bench/README.md b/bench/README.md index e5cb2e84ea7..f38fc64dc1c 100644 --- a/bench/README.md +++ b/bench/README.md @@ -924,9 +924,13 @@ the table requires no new renderer or data branch. A source's `paired.json`, as ### Paired comparison Each Nightly's Rust bench groups (`rust-bench`, `rust-shared`, `rust-relay`, -`rust-extra` and `rust-uring`) also build the previous scheduled Nightly's commit beside their -own, on the same runner, so night-to-night runner variance drops out. Each -built target runs once per side in the trend's quick mode, alternating which +`rust-extra` and `rust-uring`) also run the previous scheduled Nightly's commit beside their +own, on the same runner, so night-to-night runner variance drops out. That +night kept its bench executables as an artifact, which serve as the base +instead of a rebuild: the Rust cache cannot hold a linked executable, and +relinking `rust-bench`'s 32 fat-LTO targets took seven of its sixty minutes. +When that night kept no executable for some target, the base is built. Each +target runs once per side in the trend's quick mode, alternating which side goes first; the current side is the trend point. A target whose base executable is byte-identical to the current one is not compared: it is listed as the same build, since only noise could tell them apart. A case more than 10% diff --git a/js/json/bench/stream.ts b/js/json/bench/stream.ts index a2bdad4ed99..02b33f4ecab 100644 --- a/js/json/bench/stream.ts +++ b/js/json/bench/stream.ts @@ -21,7 +21,12 @@ for (const records of [64, 1024]) { const start = performance.now(); for (let n = 0; n < records; n++) { const record = await consumer.next(); - if (record?.value.n !== n || record.value.text !== text || record.at.isBefore(at) || at.isBefore(record.at)) { + if ( + record?.value.n !== n || + record.value.text !== text || + record.at.isBefore(at) || + at.isBefore(record.at) + ) { throw new Error("stream benchmark lost a record"); } } diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index 270764f7085..a7b8f8e8a3c 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -22,7 +22,9 @@ children for the causes nothing owned. This README's own work, after the children: -- Confirm a scheduled run on `main` passes; never dispatch Nightly. Failures +- Confirm a scheduled run on `main` passes; never dispatch Nightly. A + triage counts a cancelled job as failed: a workload that overruns fails at + its step's timeout, so a job cancelled at its own timeout hung outside it. Failures already owned elsewhere stay there: the audio near-zero p95 row belongs to [audio playout](/quest/m2/audio-playout/README.md), the macOS `receive_credit` timeout has no owner since its quest was deleted in diff --git a/quest/m0/nightly/bench-timeouts.md b/quest/m0/nightly/bench-timeouts.md deleted file mode 100644 index d84a0ba8a66..00000000000 --- a/quest/m0/nightly/bench-timeouts.md +++ /dev/null @@ -1,24 +0,0 @@ -# [S] Nightly's benchmark jobs finish within their hour - -## Goal - -Nightly's "Rust bench targets" and "JS benchmarks" jobs finish inside their -timeout and publish their series again, and the Nightly triage counts a -cancelled job as a failure, so a job that times out every night is owned. - -## Plan - -Found 2026-11-14 by the [feature review](/quest/m3/release/feature-review.md). -"Rust bench targets" was cancelled 10 of the 14 nights before 2026-11-14 -(every night since 11-09), and "JS benchmarks" 11 of 14 (every night since -11-04); in run 43601796176 the Rust job was still compiling the paired base -in `/tmp/moq-bench-base.*` when `timeout-minutes: 60` -(`.github/workflows/nightly.yml`) cancelled it. A cancelled job is not a -failed one, so the failure-based triage never named either; about 2 -job-hours a night produce nothing. Find why the paired base builds cold -though the previous Nightly built that same commit -([moq-dev/moq#8763](https://github.com/moq-dev/moq/pull/8763) notes cache -units that were "not looked up"), fix that at the cause, and only then split -or trim the jobs if they still overrun. Never raise the timeout to fit. - -Public API: none. Wire: none. diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index b207f52b37c..d3778b4f030 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -146,6 +146,8 @@ def run_group(self, group, fail="", cwd=ROOT, **environment): "#!/usr/bin/env bash\n" 'printf "%s %s\n" "${0##*/}" "$*" >> "$NIGHTLY_CALLS"\n' 'printf "%s\n" "${BENCH_BASE:-}" >> "$NIGHTLY_CALLS.base"\n' + "[[ -z ${BENCH_LABEL:-} ]] ||\n" + ' printf "%s %s\n" "$BENCH_KEEP_BUILDS" "$BENCH_BASE_BUILDS" >> "$NIGHTLY_CALLS.builds"\n' # Nextest's bench profile writes a JUnit report with a case per filtered test, # unless the test withholds the report or one test. 'if [[ ${NEXTEST_PROFILE:-} == bench && "$*" != "$NIGHTLY_NO_JUNIT" ]]; then\n' @@ -160,10 +162,17 @@ def run_group(self, group, fail="", cwd=ROOT, **environment): '[[ "$*" != "$NIGHTLY_FAIL" ]]\n' ) stub.chmod(0o755) - # Answers `gh run list` by applying its --jq filter to NIGHTLY_RUNS. + # Answers `gh run list` by applying its --jq filter to NIGHTLY_RUNS, and `gh run + # download` by logging it and failing unless NIGHTLY_KEPT is set. gh = scratch / "gh" gh.write_text( - "#!/usr/bin/env bash\nwhile [[ $1 != --jq ]]; do shift; done\n" + "#!/usr/bin/env bash\n" + "if [[ $2 == download ]]; then\n" + ' echo "$*" >> "$NIGHTLY_CALLS.download"\n' + " [[ -n ${NIGHTLY_KEPT:-} ]]\n" + " exit\n" + "fi\n" + "while [[ $1 != --jq ]]; do shift; done\n" 'printf "%s" "$NIGHTLY_RUNS" | jq -r "$2"\n' ) gh.chmod(0o755) @@ -185,6 +194,11 @@ def run_group(self, group, fail="", cwd=ROOT, **environment): raw = scratch / "results" / "raw" self.raw = sorted(path.name for path in raw.iterdir()) if raw.exists() else [] self.bases = set(Path(f"{calls}.base").read_text().splitlines()) if calls.exists() else set() + builds = Path(f"{calls}.builds") + lines = builds.read_text().splitlines() if builds.exists() else [] + self.builds = [line.split(" ") for line in lines] + download = Path(f"{calls}.download") + self.downloads = download.read_text().splitlines() if download.exists() else [] return result, calls.read_text().splitlines() if calls.exists() else [] def test_paired_base_is_the_previous_scheduled_nightly(self): @@ -215,6 +229,41 @@ def test_paired_base_is_the_previous_scheduled_nightly(self): ) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(self.bases, {head}) + # Each label keeps its executables; with no artifact named, none are fetched. + self.assertEqual(self.downloads, []) + self.assertEqual(len(self.builds), 7) + for keep, base in self.builds: + self.assertRegex(keep, r"/target/bench/builds/rust-uring/uring-\w+$") + self.assertEqual(base, "") + # The previous run's kept executables serve as each label's base. + result, _ = self.run_group( + "rust-uring", + cwd=repo, + BENCH_BASE="previous-nightly", + BENCH_BUILDS_ARTIFACT="builds-bench-rust-uring", + GITHUB_RUN_ID="2", + NIGHTLY_RUNS=runs, + NIGHTLY_KEPT="1", + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(self.downloads), 1) + self.assertRegex(self.downloads[0], r"^run download 1 --name builds-bench-rust-uring --dir \S+$") + for keep, base in self.builds: + label = keep.rsplit("/", 1)[1] + self.assertTrue(base.endswith(f"/target/bench/base-builds/rust-uring/rust-uring/{label}"), base) + # A previous run that kept none builds the base instead. + result, _ = self.run_group( + "rust-uring", + cwd=repo, + BENCH_BASE="previous-nightly", + BENCH_BUILDS_ARTIFACT="builds-bench-rust-uring", + GITHUB_RUN_ID="2", + NIGHTLY_RUNS=runs, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("kept no builds-bench-rust-uring; building the base", result.stderr) + self.assertEqual({base for _, base in self.builds}, {""}) + self.assertEqual(self.bases, {head}) # A named revision reaches the runner as its commit. result, _ = self.run_group("rust-uring", cwd=repo, BENCH_BASE="HEAD~1") self.assertEqual(result.returncode, 0, result.stderr) From abbaf853243911a7296443bbdcdae007192cd46a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 04:26:33 -0800 Subject: [PATCH 4/5] bench: build the base when a kept executable cannot load on this runner Co-Authored-By: Claude Opus 5.5 --- bench/README.md | 3 ++- bench/run.sh | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/bench/README.md b/bench/README.md index f38fc64dc1c..625f902697c 100644 --- a/bench/README.md +++ b/bench/README.md @@ -929,7 +929,8 @@ own, on the same runner, so night-to-night runner variance drops out. That night kept its bench executables as an artifact, which serve as the base instead of a rebuild: the Rust cache cannot hold a linked executable, and relinking `rust-bench`'s 32 fat-LTO targets took seven of its sixty minutes. -When that night kept no executable for some target, the base is built. Each +When that night kept no executable for some target, or one cannot load on +this runner, as after a library bump, the base is built. Each target runs once per side in the trend's quick mode, alternating which side goes first; the current side is the trend point. A target whose base executable is byte-identical to the current one is not compared: it is listed diff --git a/bench/run.sh b/bench/run.sh index e4e3f9dc58a..f0d525154b8 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -512,9 +512,10 @@ keep_builds() { } # List the base's kept executables in OUT, as criterion_build would, from the base checkout's -# packages. Fails when BENCH_BASE_BUILDS holds no builds of BASE covering every current target. +# packages. Fails when BENCH_BASE_BUILDS holds no builds of BASE covering every current target +# that can load on this host. kept_base() { - local out=$1 current=$2 dir=${BENCH_BASE_BUILDS:-} key package file + local out=$1 current=$2 dir=${BENCH_BASE_BUILDS:-} key package file loaded [[ -n $dir && -s $dir/targets.tsv && -f $dir/commit && $(<"$dir/commit") == "$BASE" ]] || return 1 dir=$(cd "$dir" && pwd -P) while IFS=$'\t' read -r key _; do @@ -524,6 +525,16 @@ kept_base() { while IFS=$'\t' read -r key package file; do # An artifact download drops the executable bit. chmod +x "$dir/$file" + # The executables link against the store paths of the shell that built them, which a + # fresh runner lacks after a toolchain or library bump. The loader alone, never main, + # resolves the interpreter and every library: anything missing builds the base instead. + if ! loaded=$(LD_TRACE_LOADED_OBJECTS=1 "$dir/$file" 2>&1); then + printf 'Kept base executable %s has no loader here\n' "$key" >&2 + return 1 + elif [[ $loaded == *"not found"* ]]; then + printf 'Kept base executable %s lacks a library here:\n%s\n' "$key" "$loaded" >&2 + return 1 + fi printf '%s\t%s\t%s\n' "$key" "$WORKTREE/$package" "$dir/$file" >>"$out" done <"$dir/targets.tsv" } From b00711a2f726e904ac8c9b254554fd41c9975496 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 15 Nov 2026 07:06:22 -0800 Subject: [PATCH 5/5] json stream bench rejects a scaled or wrong-key record timestamp isBefore does not order scaled against unscaled timestamps, so the old check accepted a scaled one. Co-Authored-By: Claude Opus 5.5 --- js/json/bench/stream.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/js/json/bench/stream.ts b/js/json/bench/stream.ts index 02b33f4ecab..355dcfb4c6e 100644 --- a/js/json/bench/stream.ts +++ b/js/json/bench/stream.ts @@ -24,8 +24,8 @@ for (const records of [64, 1024]) { if ( record?.value.n !== n || record.value.text !== text || - record.at.isBefore(at) || - at.isBefore(record.at) + record.at.scale !== undefined || + record.at.value !== at.value ) { throw new Error("stream benchmark lost a record"); }