From 6c5503dbd071b033a78a9084871cf3955c90ee62 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 17:54:24 -0800 Subject: [PATCH 1/2] chore(quest): claim quest/m0/bench-trim.md Quest-Claim: f6cf34404df88633 From a40fc58a54fd53fbff37ebe6bdb8d5c455afd487 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 9 Nov 2026 19:15:42 -0800 Subject: [PATCH 2/2] ci(bench): run the build benchmark only when its result can change Drop the nightly schedule, the moq Nix comparison, and the pure corpora (regex, wasmparser); keep the Nix table in the README. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/bench.yml | 47 +--- bench/build/README.md | 49 +--- bench/build/corpus.toml | 29 +-- bench/build/nix-report.jq | 69 ----- bench/build/nix.nix | 111 -------- bench/build/nix.sh | 267 -------------------- bench/build/nix.toml | 12 - bench/build/regex.Cargo.lock | 474 ----------------------------------- bench/build/run.sh | 29 +-- flake.lock | 22 -- flake.nix | 18 -- justfile | 6 +- quest/m0/README.md | 1 - quest/m0/bench-trim.md | 29 --- sh/dispatch.sh | 2 +- 15 files changed, 31 insertions(+), 1134 deletions(-) delete mode 100644 bench/build/nix-report.jq delete mode 100644 bench/build/nix.nix delete mode 100755 bench/build/nix.sh delete mode 100644 bench/build/nix.toml delete mode 100644 bench/build/regex.Cargo.lock delete mode 100644 quest/m0/bench-trim.md diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 6c23f7696..21f3151fc 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -4,8 +4,6 @@ permissions: contents: read on: - schedule: - - cron: "0 7 * * *" workflow_dispatch: # A change to the benchmark or its toolchains reports its numbers before it merges. pull_request: @@ -78,8 +76,8 @@ jobs: from=HEAD^ to=HEAD fi echo "Comparing $from to $to" - # The trigger's paths, less the flake's and the Nix comparison's, which runs nightly only. - if ! git diff --quiet "$from" "$to" -- bench/build .github/workflows/bench.yml ':!*.md' ':!bench/build/nix*'; then + # The trigger's paths, less the flake's. + if ! git diff --quiet "$from" "$to" -- bench/build .github/workflows/bench.yml ':!*.md'; then echo "bench=true" >>"$GITHUB_OUTPUT" exit 0 fi @@ -100,7 +98,7 @@ jobs: bench: name: Bench (${{ matrix.name }}) needs: changed - # Only the changed job's verdict skips the bench: a schedule or dispatch skips that + # Only the changed job's verdict skips the bench: a dispatch skips that # job, and its failure runs the bench rather than quietly skipping it. if: ${{ !cancelled() && needs.changed.outputs.bench != 'false' }} # x86_64 Linux is where rustc links with LLD by default. @@ -117,8 +115,6 @@ jobs: args: --corpus moq --variant nightly --variant jobs-frontend --variant cranelift --variant cache-proc-macros --variant nightly-control - name: moq ra-jit args: --corpus moq --variant ra-jit-baseline --variant ra-jit --no-attribute - - name: pure - args: --corpus regex --corpus wasmparser steps: - name: Checkout @@ -146,40 +142,3 @@ jobs: path: bench-results if-no-files-found: ignore - # moq's Nix release builds, each way they could share compilations. It takes - # hours and informs a choice rather than gating a change, so it runs nightly only. - nix: - name: Bench (moq nix) - if: github.event_name != 'pull_request' - runs-on: ubuntu-24.04 - timeout-minutes: 330 - - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/nix - - # Each variant builds in a private Nix store, which sandboxes its builds in - # user namespaces that Ubuntu reserves for root. - - name: Allow user namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - - - name: Allow perf counters - run: sudo sysctl -w kernel.perf_event_paranoid=2 - - - name: Bench - run: nix develop .#build-bench --command just bench-nix --out bench-results - - - name: Summary - run: cat bench-results/report.md >>"$GITHUB_STEP_SUMMARY" - - - name: Upload - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: bench-nix - path: bench-results - if-no-files-found: ignore diff --git a/bench/build/README.md b/bench/build/README.md index a1b493222..23e31c3e4 100644 --- a/bench/build/README.md +++ b/bench/build/README.md @@ -1,13 +1,13 @@ # Build-speed benchmark `just bench` measures Rust build time on the pinned projects and build configurations in [corpus.toml](corpus.toml). -It runs nightly in CI ([bench.yml](../../.github/workflows/bench.yml)), and on a PR that changes this harness (apart from the [Nix comparison](#nix-builds)), its workflow, the `build-bench` shell, or the `bench-ra-jit` package. +CI ([bench.yml](../../.github/workflows/bench.yml)) runs it on manual dispatch and on a PR that changes this harness, its workflow, the `build-bench` shell, or the `bench-ra-jit` package, since every input is pinned and nothing else can change its result. A later push to that PR reruns it only if it changes one of those since the PR's last benched head, so a docs-only push costs no bench time. Every build-speed change here states its effect with its numbers. ```bash just bench # everything; takes hours -just bench --corpus regex --runs 1 # a quick smoke test +just bench --variant baseline --runs 1 # a quick smoke test just bench --corpus moq --variant baseline --variant mold ``` @@ -19,9 +19,7 @@ The run also ignores the caller's `CARGO*`, `RUSTC*`, `RUSTDOC*`, `RUSTUP*`, and ## Corpus - **moq** builds `moq-relay`, which pulls in moq-net and the QUIC stack. Its crypto is on rind, which is pure Rust, so it compiles no C: the run fails if any crate in its build gains a `links` key or pulls in cc or cmake, while pure-Rust build scripts stay allowed. -- **regex** and **wasmparser** are pure Rust: the run fails if any crate in their build gains a build script or a `links` key. - -Each corpus pins a git revision (and a lockfile when the project commits none). +The corpus pins a git revision. Bump a pin in its own change, since numbers on either side of it do not compare. ## Scenarios @@ -67,50 +65,27 @@ ra-jit bugs seen on moq, not reported upstream: - Async body edits in moq-net go to cargo (every `edit-async` run): "`parsed_values_given` is not compiled by ra-jit and rustc's copy is from before the edit: an instance of it does not compile", a generic of usage-argv's. The daemon then loads the workspace again: on CI such an edit took 50.6 s and 196 CPU-s, against plain cargo's 10.5 s and 14.5 CPU-s ([run](https://github.com/moq-dev/rust2/actions/runs/42351623273)). - At the previous pin (`f36c848`), `cargo ra-jit check` with no build loaded read every crate of the workspace, took minutes and GBs, and failed on errors rustc does not report (`unresolved macro moq_clock::test` in moq-net's integration tests). -- On a library-only build such as regex, `cargo ra-jit build` returns after `cargo clean` without building anything. +- On a library-only build, `cargo ra-jit build` returns after `cargo clean` without building anything. The report shows wall time (trimmed mean, and mean ± standard deviation), CPU time and its change against the base, retired instructions, and peak RSS, all from hyperfine. A busy host adds an occasional slow run, which on an edit of a few seconds can move a mean by over 10%, so the change against a base compares trimmed means, which drop each variant's fastest and slowest run; CPU time and instructions are trimmed means too. Instructions barely move with machine load, so they are the signal on a busy machine, where wall time is only a smoke test; they read `n/a` where the host exposes no performance counters. No threshold is set: the numbers inform a decision, they do not gate one. -## Nix builds +## Nix builds for moq + +Decided on 2026-10-26: moq explores crate2nix, and keeps its crane layers as they are. -`just bench-nix` measures moq's Nix release builds (`moq-relay`, then `moq-cli`) in CPU time, built each way they could share compilations: +A since-deleted `just bench-nix` measured moq's Nix release builds (`moq-relay`, then `moq-cli`) in CPU time, built each way they could share compilations: - `crane`: moq's flake as it is, a dependency layer and a workspace layer per package. -- `crane-mbx`: the same packages with no dependency layer, every compilation going through mbx on a store mounted into the sandbox with `extra-sandbox-paths`, as nixpkgs' ccache support does. +- `crane-mbx`: the same packages with no dependency layer, every compilation going through mbx on a store mounted into the sandbox. - `crate2nix`: a derivation per crate, from `crate2nix generate`, so the Nix store itself is the cache. -- `cargo-mbx`: cargo through mbx outside Nix, as moq's CI builds today, with a fresh target directory per build. - -```bash -just bench-nix # every variant; takes hours -just bench-nix --variant crane --variant crate2nix -``` +- `cargo-mbx`: cargo through mbx outside Nix, with a fresh target directory per build. -[nix.toml](nix.toml) pins moq, the packages, the edit, and the lockfile bump; [nix.nix](nix.nix) holds the builds, with the toolchain, features, and `RUSTFLAGS` of moq's overlay throughout. -Results land in `target/bench/nix/`, newest at `target/bench/nix/latest/report.md`. -It runs nightly in CI, never on a pull request: it informs a choice rather than gating a change. - -Each variant runs four scenarios in order on a private Nix store and mbx store, so only the first is cold: - -- `clean`: the first package from nothing. -- `second-package`: the second package, which shows what the two share. -- `edit-hub`: the first package after the moq-net body edit the cargo scenarios make. -- `lock-bump`: the first package after `cargo update` of one dependency, the edit reverted. - -A cold edit or bump costs what `clean` does, so only the warm ones run. -Each Nix build runs in a store of its own, which substitutes only from cache.nixos.org, so nothing the host or moq's cache built leaks in; before the timed build, it realises everything the build needs that does not compile Rust (toolchain, sources, vendored crates). -That store runs its builds as this user in user namespaces, which makes their CPU time the `nix` process's own; Ubuntu reserves those namespaces for root unless `kernel.apparmor_restrict_unprivileged_userns` is 0. -Evaluation is timed apart from the build, since crate2nix evaluates a derivation per crate. -The report compares retired instructions, which barely move with machine load, and shows CPU time beside them. -crate2nix reads no Cargo profile, so nix.nix gives it moq's `[profile.release]`: one codegen unit, and fat LTO and symbol stripping on the linked packages. `crate2nix generate`, which a project commits, is timed but not counted. - -### Nix builds for moq - -Decided on 2026-10-26: moq explores crate2nix, and keeps its crane layers as they are. +Its scenarios ran in order on a private store, so only the first was cold: `clean` builds the first package from nothing, `second-package` the second, `edit-hub` the first after the moq-net body edit, and `lock-bump` the first after `cargo update` of one dependency. -Measured on 2026-10-25 and 2026-10-26 on a busy 32-CPU host, in billions of instructions, at the pin in nix.toml: +Measured on 2026-10-25 and 2026-10-26 on a busy 32-CPU host, in billions of instructions, at moq [`011402e`](https://github.com/moq-dev/moq/commit/011402e005f6c8efeff07c46535fa2cfa95f021e): | scenario | crane | crane-mbx | crate2nix | cargo-mbx | | --- | ---: | ---: | ---: | ---: | diff --git a/bench/build/corpus.toml b/bench/build/corpus.toml index 8322c47f3..21ffdb754 100644 --- a/bench/build/corpus.toml +++ b/bench/build/corpus.toml @@ -1,14 +1,12 @@ -# What `just bench` builds and how. Changing a rev, lock, or args is a corpus +# What `just bench` builds and how. Changing a rev or args is a corpus # bump: land it on its own so numbers on either side of it are not compared. # `args` selects what each scenario builds. `leaf` and `hub` are the crate roots # the incremental scenarios edit: a crate nothing in the build depends on, and # one most of the build depends on. `sync` and `async` are the body edits: a file # and the line in it that opens the body of a sync or async function the build -# calls; a corpus without one skips its scenario. `pure` fails the run if any -# crate in the build has a build script or links native code; `no-c` allows -# pure-Rust build scripts but fails on native links and on the cc and cmake -# crates. `test` is what the test scenarios pass to `cargo nextest run`; a corpus +# calls; a corpus without one skips its scenario. `no-c` allows pure-Rust +# build scripts but fails on native links and on the cc and cmake crates. `test` is what the test scenarios pass to `cargo nextest run`; a corpus # without it skips them. [[corpus]] @@ -44,27 +42,6 @@ test = [ "--package=qmux@0.7.0", ] -[[corpus]] -name = "regex" -git = "https://github.com/rust-lang/regex" -rev = "80b9d14e4be9f0b703a742d11c064abce05f210b" -# The repository commits no lockfile, and its dev profile optimizes to speed -# up its tests; build it at cargo's default debug opt-level instead. -lock = "regex.Cargo.lock" -args = ["-p", "regex", "--config", "profile.dev.opt-level=0"] -leaf = "src/lib.rs" -hub = "regex-syntax/src/lib.rs" -pure = true - -[[corpus]] -name = "wasmparser" -git = "https://github.com/bytecodealliance/wasm-tools" -rev = "8c7a8e6404d8f4612d975fa8943a796ecc18019c" -# The default features minus serde, whose build script makes the graph impure. -args = ["-p", "wasmparser", "--no-default-features", "--features", "std,validate,features,component-model,hash-collections,simd"] -leaf = "crates/wasmparser/src/lib.rs" -pure = true - # Each variant is one build configuration. `base` names the variant it is # compared against; a variant without one is a base. `rustflags` replaces any # host rustflags the corpus configures (none do today). `config` holds cargo diff --git a/bench/build/nix-report.jq b/bench/build/nix-report.jq deleted file mode 100644 index 4821b564e..000000000 --- a/bench/build/nix-report.jq +++ /dev/null @@ -1,69 +0,0 @@ -# Renders a `just bench-nix` run as Markdown. Inputs: $host (host.json), -# $results (results.jsonl), and $generate (crate2nix's generate timings). Run -# with `jq -n -r`. - -def fixed(p): (. * pow(10; p) | round) / pow(10; p) | tostring; -def secs: if . == null then "-" elif . < 10 then fixed(2) else fixed(1) end; -def giga: if . == null then "n/a" else (. / 1e9 | round | tostring) + "G" end; -def pct: (. * 100 | fixed(1)) as $p | if . > 0 then "+\($p)%" else "\($p)%" end; -def vs($base): if $base == null or $base == 0 or . == null then "-" else . / $base - 1 | pct end; -def count: if . == null then "-" else tostring end; -def row: "| " + join(" | ") + " |"; - -$host[0] as $h -| ["clean", "second-package", "edit-hub", "lock-bump"] as $scenarios -| ($results | map({key: "\(.scenario)/\(.variant)", value: .}) | from_entries) as $by -| ($results | map(.variant) | unique) as $variants -| [ - "# Nix build benchmark", - "", - "\($h.date) on \($h.cpu) (\($h.cpus) CPUs, \($h.memory)), \($h.os), runner `\($h.runner)`.", - "rust2 `\($h.rust2[:12])`; moq `\($h.moq[:12])`; \($h.nix); \($h.mbx); crate2nix \($h.crate2nix).", - "", - "One build per row: CPU seconds (user plus system time of every process it ran) and retired instructions.", - "Instructions barely move with machine load, so they are the comparison on a busy host; `vs crane` compares them.", - "Each variant runs the scenarios in this order on one store, so every row after `clean` starts warm.", - "Evaluation is timed apart, and the totals add it.", - "`derivations` counts the Nix derivations that compiled Rust; `compiled` counts rustc runs, as cargo's `Compiling` and `Checking` lines for crane, a derivation per crate for crate2nix, and mbx's misses for mbx; `restored` counts mbx's hits.", - "", - (["scenario", "variant", "CPU", "instructions", "vs crane", "wall", "eval CPU", "derivations", "compiled", "restored"] | row), - "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", - ( - $scenarios[] as $s - | $variants[] as $v - | $by["\($s)/\($v)"] // empty - | [ - $s, - $v, - (.build.cpu | secs), - (.build.instructions | giga), - (if $v == "crane" then "base" else .build.instructions | vs($by["\($s)/crane"].build.instructions) end), - (.build.wall | secs), - (.eval.cpu | secs), - (.derivations | count), - (.compiled | count), - (.restored | count) - ] - | row - ), - "", - (["variant", "total CPU", "total instructions", "vs crane"] | row), - "| --- | ---: | ---: | ---: |", - ( - # A variant with no evaluation adds none; a measurement a phase lacks leaves the total unknown. - def total($v; f): $results | map(select(.variant == $v) | [(.build | f), (if .eval == null then 0 else .eval | f end)]) - | flatten | if any(. == null) then null else add end; - total("crane"; .instructions) as $base - | $variants[] as $v - | [$v, (total($v; .cpu) | secs), (total($v; .instructions) | giga), - (if $v == "crane" then "base" else total($v; .instructions) | vs($base) end)] - | row - ), - ( - select($generate | length > 0) - | "", - "`crate2nix generate`, which a project reruns and commits when its lockfile changes, took " - + ($generate | map("\(.cpu | secs) CPU seconds at the \(.lock) lockfile") | join(" and ")) + "." - ) -] -| .[] diff --git a/bench/build/nix.nix b/bench/build/nix.nix deleted file mode 100644 index 5368857da..000000000 --- a/bench/build/nix.nix +++ /dev/null @@ -1,111 +0,0 @@ -# moq's Nix packages, built each way nix.sh compares. Evaluated with --impure, -# since the checkout is a path the run edits. -{ - # moq's checkout, as an absolute path string: a path literal would copy it to - # the store first. crate2nix's Cargo.nix sits at its root, beside the crates - # it names by relative path. - src, - # mbx's store path, and the sandbox path its store is mounted at. - mbx, - mbxStore, -}: -let - system = builtins.currentSystem; - # moq's own nixpkgs, rust-overlay, and crane, from its lockfile. - lock = builtins.fromJSON (builtins.readFile "${src}/flake.lock"); - locked = name: lock.nodes.${lock.nodes.root.inputs.${name}}.locked; - pkgs = import (builtins.fetchTree (locked "nixpkgs")) { - inherit system; - overlays = [ (import (builtins.fetchTree (locked "rust-overlay"))) ]; - }; - inherit (pkgs) lib; - crane' = with locked "crane"; builtins.getFlake "github:${owner}/${repo}/${rev}"; - - # The toolchain moq's overlay builds with, which pins the same channel. - channel = (lib.importTOML "${src}/rust-toolchain.toml").toolchain.channel; - toolchain = pkgs.rust-bin.stable.${channel}.default; - - # What nix/overlay.nix passes cargo for each package; crate2nix and cargo - # resolve the same features from it. - packages = { - moq-relay = [ "jemalloc" ]; - moq-cli = [ "jemalloc" ]; - }; - rustflags = [ - "-C" - "force-frame-pointers=yes" - ]; - - # crane as moq builds it: its flake's overlay, keyed by cargo package. - moq = pkgs.extend (import "${src}/nix/overlay.nix" { crane = crane'; }); - crane = lib.mapAttrs (name: _: moq.${name}) packages; - - # crane with no dependency layer: every compilation goes through mbx, on a - # store outside the sandbox. `cargo` on PATH is mbx's shim, which finds the - # real cargo after its own directory, so that directory goes on PATH as is. - craneMbx = lib.mapAttrs ( - _: drv: - drv.overrideAttrs (old: { - cargoArtifacts = null; - preConfigure = '' - export PATH=${builtins.storePath mbx}/libexec/mbx:$PATH - '' - + (old.preConfigure or ""); - # mbx would move a target directory it finds into its store. - CARGO_TARGET_DIR = "target"; - MBX_CACHE_DIR = mbxStore; - MBX_GC_AUTO = "0"; - # The report lands in the output, where nix.sh counts hits and misses. - preBuild = (old.preBuild or "") + '' - export MBX_STATS_REPORT=$NIX_BUILD_TOP/mbx-report.json - ''; - postInstall = (old.postInstall or "") + '' - install -Dm644 $MBX_STATS_REPORT $out/share/mbx/report.json - ''; - }) - ) crane; - - # One derivation per crate, with crate2nix's generated build plan. It reads no - # Cargo profile, so the packages take moq's `[profile.release]` here: - # buildRustCrate's one codegen unit already matches, and fat LTO and symbol - # stripping apply where cargo applies them, to the linked packages. - crate2nix = - let - buildRustCrate = pkgs.buildRustCrate.override { - rustc = toolchain; - cargo = toolchain; - }; - release = [ - "-C" - "lto=fat" - "-C" - "strip=symbols" - ]; - plan = pkgs.callPackage "${src}/Cargo.nix" { - buildRustCrateForPkgs = - _: crate: - buildRustCrate ( - crate - // { - extraRustcOpts = - rustflags ++ lib.optionals (packages ? ${crate.crateName}) release ++ (crate.extraRustcOpts or [ ]); - } - ); - }; - in - # Like cargo's `--features`, these add to the package's defaults. - lib.mapAttrs ( - name: features: - plan.workspaceMembers.${name}.build.override { features = [ "default" ] ++ features; } - ) packages; -in -{ - inherit - crane - craneMbx - crate2nix - toolchain - packages - rustflags - ; -} diff --git a/bench/build/nix.sh b/bench/build/nix.sh deleted file mode 100755 index f6c8e383a..000000000 --- a/bench/build/nix.sh +++ /dev/null @@ -1,267 +0,0 @@ -#!/usr/bin/env bash -# Measures moq's Nix release builds in CPU time, built each way nix.nix offers. -# Usage and output are described in README.md. -set -euo pipefail - -bench=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -root=$(cd "$bench/../.." && pwd) - -# The build-bench shell carries hyperfine, taplo, and cargo. -if [[ -z ${RUST2_BENCH_NIGHTLY:-} ]]; then - exec nix develop "$root#build-bench" --command "$bench/nix.sh" "$@" -fi - -die() { - echo "bench-nix: $*" >&2 - exit 1 -} - -usage() { - cat <<'EOF' -Usage: just bench-nix [OPTIONS] - -Options: - --variant NAME Measure only this variant (repeatable): crane, crane-mbx, - crate2nix, or cargo-mbx (default: all) - --out DIR Write results to this new or empty directory - (default: target/bench/nix/) -EOF -} - -all_variants=(crane crane-mbx crate2nix cargo-mbx) -variants=() -out= -while (($#)); do - case $1 in - --variant) variants+=("${2:?--variant needs a name}") && shift 2 ;; - --out) out=${2:?--out needs a directory} && shift 2 ;; - -h | --help) usage && exit 0 ;; - *) usage >&2 && die "unknown argument: $1" ;; - esac -done -((${#variants[@]})) || variants=("${all_variants[@]}") -for name in "${variants[@]}"; do - [[ " ${all_variants[*]} " == *" $name "* ]] || die "unknown variant: $name" -done - -config=$(taplo get -f "$bench/nix.toml" -o json) -field() { jq -r "$1" <<<"$config"; } -mapfile -t packages < <(field '.packages[]') -mapfile -t bump < <(field '.bump[]') -hub=$(field .hub) - -# Apart from run.sh's, so a `just bench` beside this run keeps its checkout. -work=/var/tmp/rust2-bench-$(id -un)/nix -results=$root/target/bench/nix -out=${out:-$results/$(date -u +%Y%m%dT%H%M%SZ)} -[[ -z $(find "$out" -mindepth 1 -print -quit 2>/dev/null) ]] || die "$out is not empty; pass a new --out" -mkdir -p "$work" "$out/raw" "$out/logs" -out=$(cd "$out" && pwd) -src=$work/src - -# As in run.sh: none of the caller's cargo, rustc, or mbx settings reach the builds. -caller_vars=("${!CARGO@}" "${!RUSTC@}" "${!RUSTDOC@}" "${!RUSTUP@}" "${!MBX@}") -for name in RUSTFLAGS CI TF_BUILD; do - [[ -z ${!name+set} ]] || caller_vars+=("$name") -done -if ((${#caller_vars[@]})); then - echo "bench-nix: ignoring ${caller_vars[*]}" - unset "${caller_vars[@]}" -fi -export CARGO_HOME=$work/cargo-home - -tools=$(nix build --no-link --print-out-paths "$root#bench-nix") -PATH=$tools/bin:$PATH -# mbx's own store path, which the Nix builds take as an input. -mbx=$(dirname "$(dirname "$(readlink -f "$tools/bin/mbx")")") - -# Each Nix variant builds in a store of its own, so it starts from nothing and -# substitutes nothing the host built. Builds run as this user, so their CPU time -# is this process's children's, in a build directory of the store's own, since -# the host's may belong to its daemon. -store_args() { # variant - printf '%s\n' --store "$work/$1/store" --option build-dir "$work/$1/build" --option substituters https://cache.nixos.org \ - --option max-jobs "$(getconf _NPROCESSORS_ONLN)" --option cores 0 \ - --option extra-sandbox-paths "/mbx=$work/$1/mbx" -} - -eval_args() { - printf '%s\n' --impure -f "$bench/nix.nix" --argstr src "$src" --argstr mbx "$mbx" --argstr mbxStore /mbx -} - -record_host() { - jq -n \ - --arg date "$(date -u +%FT%TZ)" \ - --arg os "$(uname -srm)" \ - --arg cpu "$(sed -n 's/^model name[[:space:]]*: //p' /proc/cpuinfo | head -1)" \ - --arg cpus "$(getconf _NPROCESSORS_ONLN)" \ - --arg memory "$(awk '/MemTotal/ { printf "%.0f GiB", $2 / 1048576 }' /proc/meminfo)" \ - --arg runner "${RUNNER_NAME:-local}" \ - --arg rust2 "$(git -C "$root" rev-parse HEAD)" \ - --arg moq "$(field .rev)" \ - --arg nix "$(nix --version)" \ - --arg mbx "$("$mbx/bin/mbx" --version)" \ - --arg crate2nix "$(crate2nix --version)" \ - '$ARGS.named' >"$out/host.json" -} - -# Resets the checkout to the pin, with no edit and the pinned lockfile. -checkout() { - local rev - rev=$(field .rev) - if [[ ! -d $src/.git ]]; then - git init -q "$src" - git -C "$src" remote add origin "$(field .git)" - fi - if [[ $(git -C "$src" rev-parse -q --verify HEAD || true) != "$rev" ]]; then - git -C "$src" fetch -q --depth 1 origin "$rev" - git -C "$src" checkout -q --detach FETCH_HEAD - fi - git -C "$src" reset -q --hard "$rev" - git -C "$src" clean -qfdx -} - -# Times one command with hyperfine, its output in the log. hyperfine runs a -# command without a shell, so one redirects it. -time_cmd() { # json log command... - local json=$1 log=$2 cmd - shift 2 - cmd="$(printf '%q ' "$@") >$(printf '%q' "$log") 2>&1" - hyperfine --style none --runs 1 --export-json "$json" "bash -c $(printf '%q' "$cmd")" >/dev/null || - { tail -60 "$log" >&2 && die "failed: $*"; } -} - -# One hyperfine measurement, as the results record it. -measurement() { # hyperfine.json - jq -c '.results[0].measurements[0] - | {wall: .time_wall_clock.value, cpu: .time_cpu.value, instructions: .instructions.value, rss: .memory_peak_resident.value}' "$1" -} - -# Appends one step's result. `compiled` counts the compilations that ran the -# compiler and `restored` those a cache served, each as the variant can tell. -record() { # variant scenario package derivations compiled restored eval build - jq -n -c --arg variant "$1" --arg scenario "$2" --arg package "$3" \ - --argjson derivations "$4" --argjson compiled "$5" --argjson restored "$6" --argjson eval "$7" --argjson build "$8" \ - '{$variant, $scenario, $package, $eval, $build, $derivations, $compiled, $restored}' >>"$out/results.jsonl" -} - -# The compilations an mbx build report says ran the compiler, and those it restored. -mbx_counts() { # report.json - jq -r '"\(.misses + .unconsulted) \(.hits)"' "$1" -} - -# The derivations that compile Rust: crane's, and crate2nix's one per crate. -measured() { # closure.json - jq -r '.derivations | to_entries[] | select(.value.env.cargoVendorDir != null or .value.env.crateName != null) | .key' "$1" -} - -# One step of a Nix variant: evaluate, realise what the build needs but does not -# measure (toolchain, sources, vendored crates), then time the build. -nix_step() { # variant scenario package attr - local variant=$1 scenario=$2 package=$3 attr=$4 drv output compiled restored=null tag=$1.$2 - local -a nixo args measured prereqs built - mapfile -t nixo < <(store_args "$variant") - mapfile -t args < <(eval_args) - time_cmd "$out/raw/$tag.eval.json" "$out/logs/$tag.eval.log" nix "${nixo[@]}" eval --raw "${args[@]}" "$attr.drvPath" - drv=$(<"$out/logs/$tag.eval.log") - [[ $drv == /nix/store/*.drv ]] || die "$variant $scenario evaluated to $drv" - nix "${nixo[@]}" derivation show -r "$drv" >"$work/closure.json" - mapfile -t measured < <(measured "$work/closure.json") - mapfile -t prereqs < <(jq -r --args ' - .derivations as $d - | [$ARGS.positional[] | $d[.].inputs.drvs | to_entries[] | select(.key | IN($ARGS.positional[]) | not) - | "/nix/store/\(.key)^\(.value.outputs | join(","))"] - | unique[]' "${measured[@]}" <"$work/closure.json") - if ((${#prereqs[@]})); then - nix "${nixo[@]}" build --no-link "${prereqs[@]}" >"$out/logs/$tag.prereqs.log" 2>&1 || - { tail -40 "$out/logs/$tag.prereqs.log" >&2 && die "$variant failed to realise what $scenario needs"; } - fi - # The derivations that compile Rust which the build will run. - mapfile -t built < <(nix "${nixo[@]}" build --dry-run "$drv^out" 2>&1 | - sed -n 's|^ /nix/store/\(.*\.drv\)$|\1|p' | grep -Fx -f <(printf '%s\n' "${measured[@]}") || true) - time_cmd "$out/raw/$tag.build.json" "$out/logs/$tag.build.log" nix "${nixo[@]}" build --no-link -L "$drv^out" - case $variant in - crane) compiled=$(grep -cE '^[^>]*> +(Compiling|Checking) ' "$out/logs/$tag.build.log" || true) ;; - crate2nix) compiled=${#built[@]} ;; - crane-mbx) - output=$(nix "${nixo[@]}" build --no-link --print-out-paths "$drv^out") - read -r compiled restored < <(mbx_counts "$work/$variant/store$output/share/mbx/report.json") - ;; - esac - record "$variant" "$scenario" "$package" "${#built[@]}" "$compiled" "$restored" \ - "$(measurement "$out/raw/$tag.eval.json")" "$(measurement "$out/raw/$tag.build.json")" -} - -# One step of cargo with mbx: a fresh target directory, as a CI job starts with, -# on the store earlier steps filled. -cargo_step() { # scenario package - local scenario=$1 package=$2 tag=cargo-mbx.$1 compiled restored - rm -rf "$CARGO_TARGET_DIR" "$MBX_STATS_REPORT" - ( - cd "$src" - time_cmd "$out/raw/$tag.build.json" "$out/logs/$tag.build.log" \ - mbx build --release --locked --offline -p "$package" --features "$(jq -r --arg p "$package" '.[$p] | join(",")' <<<"$features")" - ) - read -r compiled restored < <(mbx_counts "$MBX_STATS_REPORT") - record cargo-mbx "$scenario" "$package" null "$compiled" "$restored" null "$(measurement "$out/raw/$tag.build.json")" -} - -# crate2nix's build plan for the checkout. A project commits it, so its time is -# recorded apart from the builds. -generate() { - local lock tag - git -C "$src" diff --quiet -- Cargo.lock && lock=pinned || lock=bumped - tag=crate2nix.generate-$lock - (cd "$src" && time_cmd "$out/raw/$tag.json" "$out/logs/$tag.log" crate2nix generate) - measurement "$out/raw/$tag.json" | jq -c --arg lock "$lock" '{$lock} + .' >>"$out/generate.jsonl" -} - -# Runs every scenario for one variant, from an empty store. -run_variant() { # variant - local variant=$1 first=${packages[0]} second=${packages[1]} attr - echo "bench-nix: $variant" - [[ ! -e $work/$variant ]] || chmod -R u+w "$work/$variant" - rm -rf "${work:?}/$variant" - mkdir -p "$work/$variant/mbx" "$work/$variant/build" - checkout - if [[ $variant == cargo-mbx ]]; then - step() { cargo_step "$1" "$2"; } - # The toolchain, features, and flags of the Nix builds, set as a CI job would. - export MBX_CACHE_DIR=$work/$variant/mbx MBX_GC_AUTO=0 MBX_STATS_REPORT=$work/$variant/report.json \ - CARGO_TARGET_DIR=$work/$variant/target PATH=$toolchain/bin:$PATH RUSTFLAGS=$rustflags CI=true - (cd "$src" && cargo fetch --locked -q) - else - nix copy --no-check-sigs --to "$work/$variant/store" "$mbx" - attr=${variant/crane-mbx/craneMbx} - step() { nix_step "$variant" "$1" "$2" "$attr.$2"; } - fi - [[ $variant != crate2nix ]] || generate - step clean "$first" - step second-package "$second" - "$bench/run.sh" edit "$src/$hub" - step edit-hub "$first" - git -C "$src" checkout -q -- "$hub" - (cd "$src" && cargo update -q "${bump[@]}") - [[ $variant != crate2nix ]] || generate - [[ $variant != cargo-mbx ]] || (cd "$src" && cargo fetch --locked -q) - step lock-bump "$first" - # A store holds gigabytes, more than a CI runner has free for four. - chmod -R u+w "$work/$variant" - rm -rf "${work:?}/$variant" -} - -record_host -checkout -mapfile -t eval_args < <(eval_args) -toolchain=$(nix build --no-link --print-out-paths "${eval_args[@]}" toolchain) -features=$(nix eval --json "${eval_args[@]}" packages) -rustflags=$(nix eval --raw "${eval_args[@]}" rustflags --apply toString) -for variant in "${variants[@]}"; do - # A subshell keeps one variant's environment out of the next. - (run_variant "$variant") -done -jq -n -r -f "$bench/nix-report.jq" --slurpfile host "$out/host.json" --slurpfile results "$out/results.jsonl" \ - --slurpfile generate <(cat "$out/generate.jsonl" 2>/dev/null) >"$out/report.md" -mkdir -p "$results" -ln -sfn "$out" "$results/latest" -echo "bench-nix: report at $out/report.md" diff --git a/bench/build/nix.toml b/bench/build/nix.toml deleted file mode 100644 index 4ac5b108a..000000000 --- a/bench/build/nix.toml +++ /dev/null @@ -1,12 +0,0 @@ -# What `just bench-nix` builds. Changing any of it is a corpus bump: land it on -# its own so numbers on either side of it are not compared. - -# moq at its own pin, since crane's dependency layer fails to build at the moq -# corpus's rev. `packages` build in order, the second after the first. `hub` is -# the crate root the edit scenario edits, and `bump` the `cargo update` -# arguments of the lockfile scenario. -git = "https://github.com/moq-dev/moq" -rev = "011402e005f6c8efeff07c46535fa2cfa95f021e" -packages = ["moq-relay", "moq-cli"] -hub = "rs/moq-net/src/lib.rs" -bump = ["-p", "hyper", "--precise", "1.12.0"] diff --git a/bench/build/regex.Cargo.lock b/bench/build/regex.Cargo.lock deleted file mode 100644 index 5d3d69c8c..000000000 --- a/bench/build/regex.Cargo.lock +++ /dev/null @@ -1,474 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "aho-corasick" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" -dependencies = [ - "log", - "memchr", -] - -[[package]] -name = "anyhow" -version = "1.0.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" - -[[package]] -name = "arbitrary" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed" -dependencies = [ - "derive_arbitrary", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi", - "libc", - "winapi", -] - -[[package]] -name = "bstr" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" -dependencies = [ - "memchr", - "serde_core", -] - -[[package]] -name = "cfg-if" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" - -[[package]] -name = "derive_arbitrary" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b034bd7d5f032402a2479444dcc6f74e36a03f31854d41680fb240ef682a1ac" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "doc-comment" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780955b8b195a21ab8e4ac6b60dd1dbdcec1dc6c51c0617964b08c81785e12c9" - -[[package]] -name = "env_logger" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" -dependencies = [ - "atty", - "humantime", - "log", - "termcolor", -] - -[[package]] -name = "equivalent" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" - -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "rand_core", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "humantime" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" - -[[package]] -name = "indexmap" -version = "2.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" -dependencies = [ - "equivalent", - "hashbrown", -] - -[[package]] -name = "lexopt" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "803ec87c9cfb29b9d2633f20cba1f488db3fd53f2158b1024cbefb47ba05d413" - -[[package]] -name = "libc" -version = "0.2.191" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" - -[[package]] -name = "log" -version = "0.4.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" -dependencies = [ - "log", -] - -[[package]] -name = "memmap2" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b0ca1155a9ecc25aa12e8313a119f32edf0255fea2f44839c4ac0a12b2ac157" -dependencies = [ - "libc", -] - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quickcheck" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95c589f335db0f6aaa168a7cd27b1fc6920f5e1470c804f814d9cd6e62a0f70b" -dependencies = [ - "rand", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "rand" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" -dependencies = [ - "getrandom", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - -[[package]] -name = "regex" -version = "1.13.1" -dependencies = [ - "aho-corasick", - "anyhow", - "doc-comment", - "env_logger", - "memchr", - "quickcheck", - "regex-automata", - "regex-syntax", - "regex-test", -] - -[[package]] -name = "regex-automata" -version = "0.4.18" -dependencies = [ - "aho-corasick", - "anyhow", - "bstr", - "doc-comment", - "env_logger", - "log", - "memchr", - "quickcheck", - "regex-syntax", - "regex-test", -] - -[[package]] -name = "regex-cli" -version = "0.2.3" -dependencies = [ - "anyhow", - "bstr", - "lexopt", - "log", - "memmap2", - "regex", - "regex-automata", - "regex-lite", - "regex-syntax", - "tabwriter", - "textwrap", -] - -[[package]] -name = "regex-lite" -version = "0.1.9" -dependencies = [ - "anyhow", - "regex-test", -] - -[[package]] -name = "regex-syntax" -version = "0.8.11" -dependencies = [ - "arbitrary", -] - -[[package]] -name = "regex-test" -version = "0.1.1" -dependencies = [ - "anyhow", - "bstr", - "serde", - "toml", -] - -[[package]] -name = "rure" -version = "0.2.5" -dependencies = [ - "libc", - "regex", -] - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_spanned" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" -dependencies = [ - "serde", -] - -[[package]] -name = "syn" -version = "3.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "tabwriter" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce91f2f0ec87dff7e6bcbbeb267439aa1188703003c6055193c821487400432" -dependencies = [ - "unicode-width", -] - -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "textwrap" -version = "0.16.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ecfad6c3abc80a577f2b91c1e412ee57e7a060d430b553c1b0c940974ebcd49" - -[[package]] -name = "toml" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.22.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" -dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - -[[package]] -name = "unicode-ident" -version = "1.0.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" - -[[package]] -name = "unicode-width" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "winnow" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" -dependencies = [ - "memchr", -] diff --git a/bench/build/run.sh b/bench/build/run.sh index 9454b4f3f..fa264db41 100755 --- a/bench/build/run.sh +++ b/bench/build/run.sh @@ -200,9 +200,8 @@ record_host() { # Clones the corpus at its pinned rev, undoing edits an interrupted run left behind. checkout() { - local name=$1 dir=$2 rev lock + local name=$1 dir=$2 rev rev=$(field corpus "$name" .rev) - lock=$(field corpus "$name" .lock) if [[ ! -d $dir/.git ]]; then git init -q "$dir" git -C "$dir" remote add origin "$(field corpus "$name" .git)" @@ -213,25 +212,21 @@ checkout() { fi git -C "$dir" reset -q --hard "$rev" git -C "$dir" clean -qfd - [[ -z $lock ]] || cp "$bench/$lock" "$dir/Cargo.lock" (cd "$dir" && PATH=$stable_path cargo fetch --locked -q) } -# Fails if a crate in the build breaks the corpus's rule. Both forbid a `links` -# key; `pure` also forbids any build script, `no-c` only the crates that compile C. -check_graph() { # corpus rule cargo-args... - local name=$1 rule=$2 tree found - shift 2 +# Fails if a crate in the build has a `links` key or compiles C with cc or cmake. +check_no_c() { # corpus cargo-args... + local name=$1 tree found + shift tree=$(cargo tree --locked --offline "$@" -e normal,build --prefix none -f '{p}' | awk '{ sub(/^v/, "", $2); print $1 " " $2 }' | sort -u | jq -R . | jq -s .) - found=$(cargo metadata --locked --offline --format-version 1 | jq -r --argjson tree "$tree" --arg rule "$rule" ' + found=$(cargo metadata --locked --offline --format-version 1 | jq -r --argjson tree "$tree" ' .packages[] | select(("\(.name) \(.version)") as $k | $tree | index($k)) - | select(.links != null or if $rule == "pure" - then any(.targets[]; .kind | index("custom-build")) - else .name == "cc" or .name == "cmake" end) + | select(.links != null or .name == "cc" or .name == "cmake") | "\(.name) \(.version)"') - [[ -z $found ]] || die "corpus $name is marked $rule but builds: $(tr '\n' ' ' <<<"$found")" + [[ -z $found ]] || die "corpus $name is marked no-c but builds: $(tr '\n' ' ' <<<"$found")" } # Enters a variant's toolchain, flags, and target directory for the cargo calls that @@ -547,11 +542,9 @@ for corpus in "${corpora[@]}"; do dir=$work/src/$corpus checkout "$corpus" "$dir" mapfile -t args < <(jq -r --arg n "$corpus" '.corpus[] | select(.name == $n) | .args[]' <<<"$config") - for rule in pure no-c; do - if [[ $(field corpus "$corpus" ".\"$rule\"") == true ]]; then - (cd "$dir" && PATH=$stable_path check_graph "$corpus" "$rule" "${args[@]}") - fi - done + if [[ $(field corpus "$corpus" '."no-c"') == true ]]; then + (cd "$dir" && PATH=$stable_path check_no_c "$corpus" "${args[@]}") + fi bench_corpus "$corpus" "$dir" # A subshell keeps the nightly toolchain and flags from leaking into the next corpus. ((attribute == 0)) || (attribute "$corpus" "$dir") diff --git a/flake.lock b/flake.lock index b88d41d02..4f61795ad 100644 --- a/flake.lock +++ b/flake.lock @@ -18,27 +18,6 @@ "type": "github" } }, - "mbx": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1793268845, - "narHash": "sha256-V71NuTIfpeFFZALlK9kDai1DEknCXC3J7ud2rAf2MTA=", - "owner": "jdx", - "repo": "mr-boxington", - "rev": "ce14529ebbdb007c6960f0ab9dce6cdb4d369c4c", - "type": "github" - }, - "original": { - "owner": "jdx", - "ref": "v1.29.0", - "repo": "mr-boxington", - "type": "github" - } - }, "nixpkgs": { "locked": { "lastModified": 1793345420, @@ -143,7 +122,6 @@ "root": { "inputs": { "flake-utils": "flake-utils", - "mbx": "mbx", "nixpkgs": "nixpkgs", "quest": "quest", "ra-jit": "ra-jit", diff --git a/flake.nix b/flake.nix index 04c2291da..2fc28e529 100644 --- a/flake.nix +++ b/flake.nix @@ -23,12 +23,6 @@ inputs.rust-overlay.follows = "rust-overlay"; inputs.flake-utils.follows = "flake-utils"; }; - # The compiler cache the benchmark's Nix comparison measures, at the release - # moq's CI runs. Bumping it is a corpus change, like ra-jit's. - mbx = { - url = "github:jdx/mr-boxington/v1.29.0"; - inputs.nixpkgs.follows = "nixpkgs"; - }; }; outputs = @@ -39,7 +33,6 @@ rust-overlay, quest, ra-jit, - mbx, ... }: flake-utils.lib.eachSystem @@ -236,17 +229,6 @@ raJitToolchain ]; }; - } - # What `just bench-nix` runs beyond the build-bench shell, built only then: - # mbx compiles from source. - // pkgs.lib.optionalAttrs (mbx.packages ? ${system}) { - bench-nix = pkgs.symlinkJoin { - name = "bench-nix"; - paths = [ - mbx.packages.${system}.mbx - pkgs.crate2nix - ]; - }; }; # `just bench` enters this shell; the nightly toolchain stays off PATH # so stable remains the default. It holds only what the benchmark runs: diff --git a/justfile b/justfile index 0431b467b..b5de31e78 100644 --- a/justfile +++ b/justfile @@ -43,10 +43,6 @@ test $BASE="": bench *args: bench/build/run.sh {{ args }} -# Measure moq's Nix release builds in CPU time, built each way; see bench/build/README.md. -bench-nix *args: - bench/build/nix.sh {{ args }} - # Measure what moq's binding wrappers add over Rust, against the UniFFI fork; see bench/ffi/README.md. bench-ffi *args: bench/ffi/run.sh {{ args }} @@ -55,7 +51,7 @@ bench-ffi *args: fix: shfmt -w bench/*/*.sh sh/*.sh taplo format - nixfmt flake.nix bench/build/nix.nix + nixfmt flake.nix cargo fmt --all just ts fix just ffi fix diff --git a/quest/m0/README.md b/quest/m0/README.md index ab5c5f471..702253f88 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -10,4 +10,3 @@ each run: the checks, a red nightly, and code no consumer uses. - [Checks at moq's cost](/quest/m0/checks/README.md) - `just ci` selects by the diff, builds warm, and runs the differential optimized, with a nightly full `just gate` - [Delete the UniFFI fork](/quest/m0/delete-uniffi.md) - `uniffi/`, its workspace, flake packages, nightly jobs, and the benchmark harness built on it are gone - [The nightly passes](/quest/m0/nightly-green.md) - red since 2026-10-29: the wasm-bindgen fixture's NaN, Kotlin on Android API 35, and `just ffi moq` -- [Trim the build-speed benchmark](/quest/m0/bench-trim.md) - it runs on a pin change, not nightly, without the nix job and the `pure` corpora diff --git a/quest/m0/bench-trim.md b/quest/m0/bench-trim.md deleted file mode 100644 index 0ec31739c..000000000 --- a/quest/m0/bench-trim.md +++ /dev/null @@ -1,29 +0,0 @@ -# [S] Trim the build-speed benchmark - -## Goal - -The [build-speed benchmark](/bench/build/README.md) costs CI only when its result can -change: `bench.yml` runs when a pin or the benchmark itself changes and on manual -dispatch, not nightly; the `Bench (moq nix)` job and `bench/build/nix.*` are deleted, and -so are the `pure` corpora (regex, wasmparser) and their job, with every table they -produced kept in the README. - -## Plan - -Decided with the maintainer on 2026-11-09, in the quest audit: ✅ trim it, in m0 since it -is CI cost. Build speed is outside rust2's goal as the maintainer restated it, any Rust -crate callable from any language, so the track keeps only what costs nothing. Rejected: -keeping the nightly schedule and fixing the nix job; removing the track. - -- The nightly `bench.yml` costs about 3.5 runner-hours on inputs that are all pinned (the - moq revision, the nightly, ra-jit), and a pin change already runs it on its PR. -- `Bench (moq nix)` has failed every scheduled run since 2026-11-02, and the choice it - informed was made on 2026-10-26 (`bench/build/README.md`). -- The `pure` corpora were built for the subset compiler, closed on 2026-11-06 - ([m4](/quest/m4/README.md#plan)). -- What stays: the four quests that wait on upstream - ([Cranelift](/quest/m3/cranelift-unwind.md), [relink](/quest/m3/relink.md), - [TPDE](/quest/m4/tpde.md), [the fork ledger](/quest/m4/fork.md)), the benchmark they - measure with, and rust2-lint, which moq's CI plans to adopt. - -Public API: none. diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 8d6ebf370..fe86c25b7 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -91,7 +91,7 @@ check_lint() { shellcheck "${scripts[@]}" shfmt -d "${scripts[@]}" taplo format --check - nixfmt --check flake.nix bench/build/nix.nix + nixfmt --check flake.nix } check_quest() { quest check; } # The root workspace holds only the UniFFI fork.