From e89204f0bdca05f3110d80484361bc5abbe091b4 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:48:28 -0700 Subject: [PATCH 1/6] chore(quest): claim quest/m1/bench-nix.md Quest-Claim: ad8a4d2f950fbc0c From 5c773ff5a3c0375c91fd699c335adde12b0ba87d Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 25 Oct 2026 18:36:42 -0700 Subject: [PATCH 2/6] wip(bench): Nix build comparison harness Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0115AT2ue6x9cT5e5J9SZtCn --- bench/build/nix-report.jq | 59 +++++++++ bench/build/nix.nix | 96 +++++++++++++++ bench/build/nix.sh | 245 ++++++++++++++++++++++++++++++++++++++ bench/build/nix.toml | 12 ++ flake.lock | 22 ++++ flake.nix | 18 +++ justfile | 4 + 7 files changed, 456 insertions(+) create mode 100644 bench/build/nix-report.jq create mode 100644 bench/build/nix.nix create mode 100755 bench/build/nix.sh create mode 100644 bench/build/nix.toml diff --git a/bench/build/nix-report.jq b/bench/build/nix-report.jq new file mode 100644 index 000000000..70e42182a --- /dev/null +++ b/bench/build/nix-report.jq @@ -0,0 +1,59 @@ +# Renders a `just bench-nix` run as Markdown. Inputs: $host (host.json), +# $results (results.jsonl), and $generate (crate2nix's generate timings). Run +# with `jq -n -r`. + +def fixed(p): (. * pow(10; p) | round) / pow(10; p) | tostring; +def secs: if . == null then "-" elif . < 10 then fixed(2) else fixed(1) end; +def pct: (. * 100 | fixed(1)) as $p | if . > 0 then "+\($p)%" else "\($p)%" end; +def row: "| " + join(" | ") + " |"; +def scenarios: ["clean", "second-package", "edit-hub", "lock-bump"]; + +$host[0] as $h +| ($results | map({key: "\(.scenario)/\(.variant)", value: .}) | from_entries) as $by +| ($results | map(.variant) | unique) as $variants +| [ + "# Nix build benchmark", + "", + "\($h.date) on \($h.cpu) (\($h.cpus) CPUs, \($h.memory)), \($h.os), runner `\($h.runner)`.", + "rust2 `\($h.rust2[:12])`; moq `\($h.moq[:12])`; \($h.nix); \($h.mbx); \($h.crate2nix).", + "", + "CPU and wall seconds of one build each, user plus system time of every process it ran.", + "Each variant runs the scenarios in order on one store, so each after `clean` starts warm.", + "`vs crane` compares CPU time; evaluation is timed apart from the build.", + "`derivations` counts the Nix derivations that compiled Rust, and `compiled` the crates rustc compiled:", + "cargo's `Compiling` and `Checking` lines for crane, a derivation per crate for crate2nix, and mbx's misses where mbx ran.", + "", + (["scenario", "variant", "CPU", "vs crane", "wall", "eval CPU", "derivations", "compiled", "mbx hits"] | row), + "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + ( + scenarios[] as $s + | $variants[] as $v + | $by["\($s)/\($v)"] // empty + | $by["\($s)/crane"] as $base + | [ + $s, + $v, + (.build.cpu | secs), + (if $v == "crane" then "base" elif $base == null then "-" else .build.cpu / $base.build.cpu - 1 | pct end), + (.build.wall | secs), + (.eval.cpu | secs), + (.derivations // "-" | tostring), + (.mbx.misses // .compiled | tostring), + (.mbx.hits // "-" | tostring) + ] + | row + ), + ( + $variants[] as $v + | [$results[] | select(.variant == $v) | .build.cpu + (.eval.cpu // 0)] | add + | [$v, "all scenarios", (. | secs)] + | "", + "Total \(.[0]): \(.[2]) CPU seconds over \(.[1]), evaluation included." + ), + ( + $generate[] + | "", + "`crate2nix generate`, which a project reruns when its lockfile changes and commits, took \(.cpu | secs) CPU seconds and \(.wall | secs) wall at the \(.lock) lockfile." + ) +] +| .[] diff --git a/bench/build/nix.nix b/bench/build/nix.nix new file mode 100644 index 000000000..5da8cfb1d --- /dev/null +++ b/bench/build/nix.nix @@ -0,0 +1,96 @@ +# moq's Nix packages, built each way nix.sh compares. Evaluated with --impure, +# since the checkout is a path the run edits. +{ + # moq's checkout, as an absolute path string: a path literal would copy it to + # the store first. crate2nix's Cargo.nix sits at its root, beside the crates + # it names by relative path. + src, + # mbx's store path, and the sandbox path its store is mounted at. + mbx, + mbxStore, +}: +let + # Only crane's variants evaluate moq's flake, which copies the whole checkout. + moq = builtins.getFlake "path:${src}"; + system = builtins.currentSystem; + # moq's own nixpkgs and rust-overlay, from its lockfile. + lock = builtins.fromJSON (builtins.readFile "${src}/flake.lock"); + input = name: builtins.fetchTree lock.nodes.${lock.nodes.root.inputs.${name}}.locked; + pkgs = import (input "nixpkgs") { + inherit system; + overlays = [ (import (input "rust-overlay")) ]; + }; + inherit (pkgs) lib; + + # The toolchain moq's overlay builds with, which pins the same channel. + channel = (lib.importTOML "${src}/rust-toolchain.toml").toolchain.channel; + toolchain = pkgs.rust-bin.stable.${channel}.default; + + # What nix/overlay.nix passes cargo for each package; crate2nix and cargo + # resolve the same features from it. + packages = { + moq-relay = [ "jemalloc" ]; + moq-cli = [ "jemalloc" ]; + }; + rustflags = [ + "-C" + "force-frame-pointers=yes" + ]; + + # crane as moq builds it. + crane = lib.mapAttrs (name: _: moq.packages.${system}.${name}) packages; + + # crane with no dependency layer: every compilation goes through mbx, on a + # store outside the sandbox. `cargo` on PATH is mbx's shim. + mbxCargo = pkgs.runCommand "mbx-cargo" { } '' + mkdir -p $out/bin + ln -s ${builtins.storePath mbx}/libexec/mbx/cargo $out/bin/cargo + ''; + craneMbx = lib.mapAttrs ( + _: drv: + drv.overrideAttrs (old: { + cargoArtifacts = null; + nativeBuildInputs = [ mbxCargo ] ++ old.nativeBuildInputs; + # mbx would move a target directory it finds into its store. + CARGO_TARGET_DIR = "target"; + MBX_CACHE_DIR = mbxStore; + MBX_GC_AUTO = "0"; + # The report lands in the output, where nix.sh counts hits and misses. + preBuild = (old.preBuild or "") + '' + export MBX_STATS_REPORT=$NIX_BUILD_TOP/mbx-report.json + ''; + postInstall = (old.postInstall or "") + '' + install -Dm644 $MBX_STATS_REPORT $out/share/mbx/report.json + ''; + }) + ) crane; + + # One derivation per crate, with crate2nix's generated build plan. + crate2nix = + let + buildRustCrate = pkgs.buildRustCrate.override { + rustc = toolchain; + cargo = toolchain; + # Cargo's release default, which the other ways build with; buildRustCrate's is 1. + defaultCodegenUnits = 16; + defaultCrateOverrides = pkgs.defaultCrateOverrides // overrides; + }; + # Crates whose build scripts need more than nixpkgs' overrides give them. + overrides = { }; + plan = pkgs.callPackage "${src}/Cargo.nix" { + buildRustCrateForPkgs = + _: crate: buildRustCrate (crate // { extraRustcOpts = rustflags ++ (crate.extraRustcOpts or [ ]); }); + }; + in + lib.mapAttrs (name: features: plan.workspaceMembers.${name}.build.override { inherit features; }) packages; +in +{ + inherit + crane + craneMbx + crate2nix + toolchain + packages + rustflags + ; +} diff --git a/bench/build/nix.sh b/bench/build/nix.sh new file mode 100755 index 000000000..330f1aaf3 --- /dev/null +++ b/bench/build/nix.sh @@ -0,0 +1,245 @@ +#!/usr/bin/env bash +# Measures moq's Nix release builds, built each way nix.nix offers, in CPU time. +# Usage and output are described in README.md. +set -euo pipefail + +bench=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +root=$(cd "$bench/../.." && pwd) + +# The build-bench shell carries hyperfine and crate2nix. +if [[ -z ${RUST2_BENCH_NIGHTLY:-} ]]; then + exec nix develop "$root#build-bench" --command "$bench/nix.sh" "$@" +fi + +die() { + echo "bench-nix: $*" >&2 + exit 1 +} + +usage() { + cat <<'EOF' +Usage: just bench-nix [OPTIONS] + +Options: + --variant NAME Measure only this variant (repeatable): crane, crane-mbx, + crate2nix, or cargo-mbx (default: all) + --out DIR Write results to this new or empty directory + (default: target/bench/nix/) +EOF +} + +all_variants=(crane crane-mbx crate2nix cargo-mbx) +variants=() +out= +while (($#)); do + case $1 in + --variant) variants+=("${2:?--variant needs a name}") && shift 2 ;; + --out) out=${2:?--out needs a directory} && shift 2 ;; + -h | --help) usage && exit 0 ;; + *) usage >&2 && die "unknown argument: $1" ;; + esac +done +((${#variants[@]})) || variants=("${all_variants[@]}") +for name in "${variants[@]}"; do + [[ " ${all_variants[*]} " == *" $name "* ]] || die "unknown variant: $name" +done + +config=$(taplo get -f "$bench/nix.toml" -o json) +field() { jq -r "$1" <<<"$config"; } +mapfile -t packages < <(field '.packages[]') +mapfile -t bump < <(field '.bump[]') +hub=$(field .hub) + +# Private to this script, so a `just bench` running beside it keeps its checkout. +work=/var/tmp/rust2-bench-$(id -un)/nix +results=$root/target/bench/nix +out=${out:-$results/$(date -u +%Y%m%dT%H%M%SZ)} +[[ -z $(find "$out" -mindepth 1 -print -quit 2>/dev/null) ]] || die "$out is not empty; pass a new --out" +mkdir -p "$work" "$out/raw" "$out/logs" +out=$(cd "$out" && pwd) +src=$work/src + +# As in run.sh: none of the caller's cargo or rustc settings reach the builds. +caller_vars=("${!CARGO@}" "${!RUSTC@}" "${!RUSTDOC@}" "${!RUSTUP@}" "${!MBX@}") +for name in RUSTFLAGS CI TF_BUILD; do + [[ -z ${!name+set} ]] || caller_vars+=("$name") +done +if ((${#caller_vars[@]})); then + echo "bench-nix: ignoring ${caller_vars[*]}" + unset "${caller_vars[@]}" +fi +export CARGO_HOME=$work/cargo-home + +tools=$(nix build --no-link --print-out-paths "$root#bench-nix") +PATH=$tools/bin:$PATH +# mbx's own store path, which the Nix builds take as an input. +mbx=$(dirname "$(dirname "$(readlink -f "$tools/bin/mbx")")") + +# Every Nix variant builds in a store of its own under the work directory, so a +# run starts from nothing and nothing the host built is substituted. Builds run +# as this user, so their CPU time is this process's children's. Each runs on +# every core, with cargo's release codegen units. +store_args() { # variant + printf '%s\n' --store "$work/$1/store" --option substituters https://cache.nixos.org \ + --option max-jobs "$(getconf _NPROCESSORS_ONLN)" --option cores 0 \ + --option extra-sandbox-paths "/mbx=$work/$1/mbx" +} + +eval_args() { + printf '%s\n' --impure -f "$bench/nix.nix" --argstr src "$src" --argstr mbx "$mbx" --argstr mbxStore /mbx +} + +record_host() { + jq -n \ + --arg date "$(date -u +%FT%TZ)" \ + --arg os "$(uname -srm)" \ + --arg cpu "$(sed -n 's/^model name[[:space:]]*: //p' /proc/cpuinfo | head -1)" \ + --arg cpus "$(getconf _NPROCESSORS_ONLN)" \ + --arg memory "$(awk '/MemTotal/ { printf "%.0f GiB", $2 / 1048576 }' /proc/meminfo)" \ + --arg runner "${RUNNER_NAME:-local}" \ + --arg rust2 "$(git -C "$root" rev-parse HEAD)" \ + --arg moq "$(field .rev)" \ + --arg nix "$(nix --version)" \ + --arg mbx "$("$mbx/bin/mbx" --version)" \ + --arg crate2nix "$(crate2nix --version)" \ + '$ARGS.named' >"$out/host.json" +} + +# Resets the checkout to the pin, with no edit and the pinned lockfile. +checkout() { + local rev + rev=$(field .rev) + if [[ ! -d $src/.git ]]; then + git init -q "$src" + git -C "$src" remote add origin "$(field .git)" + fi + if [[ $(git -C "$src" rev-parse -q --verify HEAD || true) != "$rev" ]]; then + git -C "$src" fetch -q --depth 1 origin "$rev" + git -C "$src" checkout -q --detach FETCH_HEAD + fi + git -C "$src" reset -q --hard "$rev" + git -C "$src" clean -qfdx +} + +# Times one command, appending hyperfine's measurement to the step's record. +time_cmd() { # json log command... + local json=$1 log=$2 + shift 2 + hyperfine --style none --runs 1 --output "$log" --export-json "$json" "$(printf '%q ' "$@")" >/dev/null || + { tail -60 "$log" >&2 && die "failed: $*"; } +} + +# The derivations that compile Rust: crane's, and crate2nix's one per crate. +measured() { # closure.json + jq -r '.derivations | to_entries[] | select(.value.env.cargoVendorDir != null or .value.env.crateName != null) | .key' "$1" +} + +# One step of a Nix variant: evaluate, realise what the build needs but does not +# measure (toolchain, sources, vendored crates), then time the build. +nix_step() { # variant scenario attr + local variant=$1 scenario=$2 attr=$3 drv tag=$1.$2 + local -a nixo args measured prereqs built + mapfile -t nixo < <(store_args "$variant") + mapfile -t args < <(eval_args) + time_cmd "$out/raw/$tag.eval.json" "$out/logs/$tag.eval.log" nix "${nixo[@]}" eval --raw "${args[@]}" "$attr.drvPath" + drv=$(nix "${nixo[@]}" eval --raw "${args[@]}" "$attr.drvPath") + nix "${nixo[@]}" derivation show -r "$drv" >"$work/closure.json" + mapfile -t measured < <(measured "$work/closure.json") + mapfile -t prereqs < <(jq -r --args ' + .derivations as $d + | [$ARGS.positional[] | $d[.].inputs.drvs | to_entries[] | select(.key | IN($ARGS.positional[]) | not) + | "/nix/store/\(.key)^\(.value.outputs | join(","))"] + | unique[]' "${measured[@]}" <"$work/closure.json") + nix "${nixo[@]}" build --no-link "${prereqs[@]}" >"$out/logs/$tag.prereqs.log" 2>&1 || + { tail -40 "$out/logs/$tag.prereqs.log" >&2 && die "$variant failed to realise what $scenario needs"; } + # What the build will run, of the derivations that compile Rust. + mapfile -t built < <(nix "${nixo[@]}" build --dry-run --json "$drv^out" 2>&1 >/dev/null | + sed -n 's|^ /nix/store/\(.*\.drv\)$|\1|p' | grep -Fx -f <(printf '%s\n' "${measured[@]}") || true) + time_cmd "$out/raw/$tag.build.json" "$out/logs/$tag.build.log" nix "${nixo[@]}" build --no-link -L "$drv^out" + jq -n -c --arg variant "$variant" --arg scenario "$scenario" --arg attr "$attr" \ + --argjson derivations "${#built[@]}" \ + --argjson compiled "$(grep -cE '^[^>]*> +(Compiling|Checking) |^[^>]*> Building ' "$out/logs/$tag.build.log" || true)" \ + --slurpfile eval "$out/raw/$tag.eval.json" --slurpfile build "$out/raw/$tag.build.json" \ + --slurpfile mbx <(if [[ $variant == crane-mbx ]]; then + cat "$(nix "${nixo[@]}" build --no-link --print-out-paths "$drv^out" | sed 's|^/|'"$work/$variant/store"'/|')/share/mbx/report.json" + fi) ' + def m: .[0].results[0] | {wall: .mean, cpu: (.user + .system), rss: .memory_usage_byte[0]}; + {$variant, $scenario, $attr, eval: ($eval | m), build: ($build | m), $derivations, $compiled, + mbx: ($mbx[0] // null | if . then {hits, misses} else null end)}' >>"$out/results.jsonl" +} + +# One step of cargo with mbx: a fresh target directory, as a CI job starts with, +# on the store earlier steps filled. +cargo_step() { # scenario package + local scenario=$1 package=$2 tag=cargo-mbx.$1 + rm -rf "$CARGO_TARGET_DIR" + ( + cd "$src" + time_cmd "$out/raw/$tag.build.json" "$out/logs/$tag.build.log" \ + mbx build --release --locked --offline -p "$package" --features "$(jq -r --arg p "$package" '.[$p] | join(",")' <<<"$features")" + ) + jq -n -c --arg scenario "$scenario" --arg attr "$package" \ + --argjson compiled "$(grep -cE '^ +Compiling ' "$out/logs/$tag.build.log" || true)" \ + --slurpfile build "$out/raw/$tag.build.json" --slurpfile mbx "$MBX_STATS_REPORT" ' + def m: .[0].results[0] | {wall: .mean, cpu: (.user + .system), rss: .memory_usage_byte[0]}; + {variant: "cargo-mbx", $scenario, $attr, eval: null, build: ($build | m), derivations: null, $compiled, + mbx: ($mbx[0] | {hits, misses})}' >>"$out/results.jsonl" +} + +# Runs every scenario for one variant, from an empty store. +run_variant() { # variant + local variant=$1 first=${packages[0]} second=${packages[1]} step + echo "bench-nix: $variant" + chmod -R u+w "$work/$variant" 2>/dev/null || true + rm -rf "${work:?}/$variant" + mkdir -p "$work/$variant/mbx" + checkout + if [[ $variant == cargo-mbx ]]; then + step() { cargo_step "$1" "$2"; } + # The toolchain, features, and flags of the Nix builds, as a CI job would set them. + export MBX_CACHE_DIR=$work/$variant/mbx MBX_GC_AUTO=0 MBX_STATS_REPORT=$work/$variant/report.json \ + CARGO_TARGET_DIR=$work/$variant/target PATH=$toolchain/bin:$mbx/bin:$PATH RUSTFLAGS=$rustflags CI=true + (cd "$src" && cargo fetch --locked -q) + else + nix copy --no-check-sigs --to "$work/$variant/store" "$mbx" + local attr=${variant/crane-mbx/craneMbx} + step() { nix_step "$variant" "$1" "$attr.$2"; } + fi + [[ $variant != crate2nix ]] || generate + step clean "$first" + step second-package "$second" + "$bench/run.sh" edit "$src/$hub" + step edit-hub "$first" + git -C "$src" checkout -q -- "$hub" + (cd "$src" && cargo update -q "${bump[@]}") + [[ $variant != crate2nix ]] || generate + [[ $variant != cargo-mbx ]] || (cd "$src" && cargo fetch --locked -q) + step lock-bump "$first" + unset -f step +} + +# crate2nix's build plan for the checkout. A project commits it, so its time is +# recorded but not measured as a build. +generate() { + local lock tag + git -C "$src" diff --quiet -- Cargo.lock && lock=pinned || lock=bumped + tag=crate2nix.generate-$lock + (cd "$src" && time_cmd "$out/raw/$tag.json" "$out/logs/$tag.log" crate2nix generate) + jq -c --arg lock "$lock" '.results[0] | {$lock, wall: .mean, cpu: (.user + .system)}' "$out/raw/$tag.json" >>"$out/generate.jsonl" +} + +record_host +checkout +mapfile -t eval_args < <(eval_args) +toolchain=$(nix build --no-link --print-out-paths "${eval_args[@]}" toolchain) +features=$(nix eval --json "${eval_args[@]}" packages) +rustflags=$(nix eval --raw "${eval_args[@]}" rustflags --apply toString) +for variant in "${variants[@]}"; do + # A subshell, so one variant's environment stays out of the next. + (run_variant "$variant") +done +jq -n -r -f "$bench/nix-report.jq" --slurpfile host "$out/host.json" --slurpfile results "$out/results.jsonl" \ + --slurpfile generate <(cat "$out/generate.jsonl" 2>/dev/null) >"$out/report.md" +mkdir -p "$results" +ln -sfn "$out" "$results/latest" +echo "bench-nix: report at $out/report.md" diff --git a/bench/build/nix.toml b/bench/build/nix.toml new file mode 100644 index 000000000..4ac5b108a --- /dev/null +++ b/bench/build/nix.toml @@ -0,0 +1,12 @@ +# What `just bench-nix` builds. Changing any of it is a corpus bump: land it on +# its own so numbers on either side of it are not compared. + +# moq at its own pin, since crane's dependency layer fails to build at the moq +# corpus's rev. `packages` build in order, the second after the first. `hub` is +# the crate root the edit scenario edits, and `bump` the `cargo update` +# arguments of the lockfile scenario. +git = "https://github.com/moq-dev/moq" +rev = "011402e005f6c8efeff07c46535fa2cfa95f021e" +packages = ["moq-relay", "moq-cli"] +hub = "rs/moq-net/src/lib.rs" +bump = ["-p", "hyper", "--precise", "1.12.0"] diff --git a/flake.lock b/flake.lock index 4711dfb6e..84196d9e4 100644 --- a/flake.lock +++ b/flake.lock @@ -18,6 +18,27 @@ "type": "github" } }, + "mbx": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1792405127, + "narHash": "sha256-bPAAaDjorIe/9V3sgnC85yOQVerd079HT6bOfB0WgFo=", + "owner": "jdx", + "repo": "mr-boxington", + "rev": "2ade2bc6b661f1b40a55cfa7083fa9a4efc9348d", + "type": "github" + }, + "original": { + "owner": "jdx", + "ref": "v1.25.0", + "repo": "mr-boxington", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1792506237, @@ -88,6 +109,7 @@ "root": { "inputs": { "flake-utils": "flake-utils", + "mbx": "mbx", "nixpkgs": "nixpkgs", "quest": "quest", "ra-jit": "ra-jit", diff --git a/flake.nix b/flake.nix index 2e40ba9c3..0b6a2dd11 100644 --- a/flake.nix +++ b/flake.nix @@ -23,6 +23,12 @@ inputs.rust-overlay.follows = "rust-overlay"; inputs.flake-utils.follows = "flake-utils"; }; + # The compiler cache the benchmark's Nix comparison measures, at the release + # moq's CI runs. Bumping it is a corpus change, like ra-jit's. + mbx = { + url = "github:jdx/mr-boxington/v1.25.0"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = @@ -33,6 +39,7 @@ rust-overlay, quest, ra-jit, + mbx, ... }: flake-utils.lib.eachSystem @@ -169,6 +176,17 @@ raJitToolchain ]; }; + } + # What `just bench-nix` runs beyond the build-bench shell, built only then: + # mbx compiles from source. + // pkgs.lib.optionalAttrs (mbx.packages ? ${system}) { + bench-nix = pkgs.symlinkJoin { + name = "bench-nix"; + paths = [ + mbx.packages.${system}.mbx + pkgs.crate2nix + ]; + }; }; # `just bench` enters this shell; the nightly toolchain stays off PATH # so stable remains the default. It holds only what the benchmark runs: diff --git a/justfile b/justfile index a7db08e53..44d35af83 100644 --- a/justfile +++ b/justfile @@ -38,6 +38,10 @@ test: bench *args: bench/build/run.sh {{ args }} +# Measure moq's Nix release builds in CPU time, built each way; see bench/build/README.md. +bench-nix *args: + bench/build/nix.sh {{ args }} + # Measure what moq's binding wrappers add over Rust, against the UniFFI fork; see bench/ffi/README.md. bench-ffi *args: bench/ffi/run.sh {{ args }} From 482ef903cc9050d59c48a95dd3c1bb70bc5a1377 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:52:37 -0700 Subject: [PATCH 3/6] wip(bench): run the Nix comparison nightly, from moq's overlay Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0115AT2ue6x9cT5e5J9SZtCn --- .github/workflows/bench.yml | 42 +++++++++++- bench/build/README.md | 31 +++++++++ bench/build/nix-report.jq | 42 ++++++------ bench/build/nix.nix | 38 +++++------ bench/build/nix.sh | 129 +++++++++++++++++++++--------------- justfile | 4 +- 6 files changed, 190 insertions(+), 96 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index b853eeea8..1387d76b1 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -45,8 +45,8 @@ jobs: - name: Diff id: diff run: | - # The trigger's paths, less the flake's. - if ! git diff --quiet HEAD^ HEAD -- bench/build .github/workflows/bench.yml ':!*.md'; then + # The trigger's paths, less the flake's and the Nix comparison's, which runs nightly only. + if ! git diff --quiet HEAD^ HEAD -- bench/build .github/workflows/bench.yml ':!*.md' ':!bench/build/nix*'; then echo "bench=true" >>"$GITHUB_OUTPUT" exit 0 fi @@ -113,3 +113,41 @@ jobs: name: bench-${{ strategy.job-index }} path: bench-results if-no-files-found: ignore + + # moq's Nix release builds, each way they could share compilations. It takes + # hours and informs a choice rather than gating a change, so it runs nightly only. + nix: + name: Bench (moq nix) + if: github.event_name != 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 330 + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/nix + + # Each variant builds in a private Nix store, which sandboxes its builds in + # user namespaces that Ubuntu reserves for root. + - name: Allow user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Allow perf counters + run: sudo sysctl -w kernel.perf_event_paranoid=2 + + - name: Bench + run: nix develop .#build-bench --command just bench-nix --out bench-results + + - name: Summary + run: cat bench-results/report.md >>"$GITHUB_STEP_SUMMARY" + + - name: Upload + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: bench-nix + path: bench-results + if-no-files-found: ignore diff --git a/bench/build/README.md b/bench/build/README.md index 7a6b414da..4437d18c5 100644 --- a/bench/build/README.md +++ b/bench/build/README.md @@ -62,6 +62,37 @@ A busy host adds an occasional slow run, which on an edit of a few seconds can m Instructions barely move with machine load, so they are the signal on a busy machine, where wall time is only a smoke test; they read `n/a` where the host exposes no performance counters. No threshold is set: the numbers inform a decision, they do not gate one. +## Nix builds + +`just bench-nix` measures moq's Nix release builds (`moq-relay`, then `moq-cli`) in CPU time, built each way they could share compilations: + +- `crane`: moq's flake as it is, a dependency layer and a workspace layer per package. +- `crane-mbx`: the same packages with no dependency layer, every compilation going through mbx on a store mounted into the sandbox with `extra-sandbox-paths`, as nixpkgs' ccache support does. +- `crate2nix`: a derivation per crate, from `crate2nix generate`, so the Nix store itself is the cache. +- `cargo-mbx`: cargo through mbx outside Nix, as moq's CI builds today, with a fresh target directory per build. + +```bash +just bench-nix # every variant; takes hours +just bench-nix --variant crane --variant crate2nix +``` + +[nix.toml](nix.toml) pins moq, the packages, the edit, and the lockfile bump; [nix.nix](nix.nix) holds the builds, with the toolchain, features, and `RUSTFLAGS` of moq's overlay throughout. +Results land in `target/bench/nix/`, newest at `target/bench/nix/latest/report.md`. +It runs nightly in CI, never on a pull request: it informs a choice rather than gating a change. + +Each variant runs four scenarios in order on a private Nix store and mbx store, so only the first is cold: + +- `clean`: the first package from nothing. +- `second-package`: the second package, which shows what the two share. +- `edit-hub`: the first package after the moq-net body edit the cargo scenarios make. +- `lock-bump`: the first package after `cargo update` of one dependency, the edit reverted. + +A cold edit or bump costs what `clean` does, so only the warm ones run. +Each Nix build runs in a store of its own, which substitutes only from cache.nixos.org, so nothing the host or moq's cache built leaks in; before the timed build, it realises everything the build needs that does not compile Rust (toolchain, sources, vendored crates). +That store runs its builds as this user in user namespaces, which makes their CPU time the `nix` process's own; Ubuntu reserves those namespaces for root unless `kernel.apparmor_restrict_unprivileged_userns` is 0. +Evaluation is timed apart from the build, since crate2nix evaluates a derivation per crate. +crate2nix takes cargo's release codegen units rather than its default of one; `crate2nix generate`, which a project commits, is timed but not counted. + ## Attribution After the timed variants, one nightly build per corpus (clean, then each edit) records cargo's build analysis log with section timings and rustc's self-profile. diff --git a/bench/build/nix-report.jq b/bench/build/nix-report.jq index 70e42182a..1241eb5d1 100644 --- a/bench/build/nix-report.jq +++ b/bench/build/nix-report.jq @@ -5,28 +5,28 @@ def fixed(p): (. * pow(10; p) | round) / pow(10; p) | tostring; def secs: if . == null then "-" elif . < 10 then fixed(2) else fixed(1) end; def pct: (. * 100 | fixed(1)) as $p | if . > 0 then "+\($p)%" else "\($p)%" end; +def count: if . == null then "-" else tostring end; def row: "| " + join(" | ") + " |"; -def scenarios: ["clean", "second-package", "edit-hub", "lock-bump"]; $host[0] as $h +| ["clean", "second-package", "edit-hub", "lock-bump"] as $scenarios | ($results | map({key: "\(.scenario)/\(.variant)", value: .}) | from_entries) as $by | ($results | map(.variant) | unique) as $variants | [ "# Nix build benchmark", "", "\($h.date) on \($h.cpu) (\($h.cpus) CPUs, \($h.memory)), \($h.os), runner `\($h.runner)`.", - "rust2 `\($h.rust2[:12])`; moq `\($h.moq[:12])`; \($h.nix); \($h.mbx); \($h.crate2nix).", + "rust2 `\($h.rust2[:12])`; moq `\($h.moq[:12])`; \($h.nix); \($h.mbx); crate2nix \($h.crate2nix).", "", - "CPU and wall seconds of one build each, user plus system time of every process it ran.", - "Each variant runs the scenarios in order on one store, so each after `clean` starts warm.", - "`vs crane` compares CPU time; evaluation is timed apart from the build.", - "`derivations` counts the Nix derivations that compiled Rust, and `compiled` the crates rustc compiled:", - "cargo's `Compiling` and `Checking` lines for crane, a derivation per crate for crate2nix, and mbx's misses where mbx ran.", + "One build per row, in CPU seconds: user plus system time of every process it ran.", + "Each variant runs the scenarios in this order on one store, so every row after `clean` starts warm.", + "`vs crane` compares CPU time. Evaluation is timed apart, and the total adds it.", + "`derivations` counts the Nix derivations that compiled Rust; `compiled` counts rustc runs, as cargo's `Compiling` and `Checking` lines for crane, a derivation per crate for crate2nix, and mbx's misses for mbx; `restored` counts mbx's hits.", "", - (["scenario", "variant", "CPU", "vs crane", "wall", "eval CPU", "derivations", "compiled", "mbx hits"] | row), + (["scenario", "variant", "CPU", "vs crane", "wall", "eval CPU", "derivations", "compiled", "restored"] | row), "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", ( - scenarios[] as $s + $scenarios[] as $s | $variants[] as $v | $by["\($s)/\($v)"] // empty | $by["\($s)/crane"] as $base @@ -37,23 +37,27 @@ $host[0] as $h (if $v == "crane" then "base" elif $base == null then "-" else .build.cpu / $base.build.cpu - 1 | pct end), (.build.wall | secs), (.eval.cpu | secs), - (.derivations // "-" | tostring), - (.mbx.misses // .compiled | tostring), - (.mbx.hits // "-" | tostring) + (.derivations | count), + (.compiled | count), + (.restored | count) ] | row ), + "", + (["variant", "total CPU", "vs crane"] | row), + "| --- | ---: | ---: |", ( - $variants[] as $v - | [$results[] | select(.variant == $v) | .build.cpu + (.eval.cpu // 0)] | add - | [$v, "all scenarios", (. | secs)] - | "", - "Total \(.[0]): \(.[2]) CPU seconds over \(.[1]), evaluation included." + ($results | map(select(.variant == "crane") | .build.cpu + .eval.cpu) | add) as $base + | $variants[] as $v + | ($results | map(select(.variant == $v) | .build.cpu + (.eval.cpu // 0)) | add) as $total + | [$v, ($total | secs), (if $v == "crane" then "base" elif $base == null then "-" else $total / $base - 1 | pct end)] + | row ), ( - $generate[] + select($generate | length > 0) | "", - "`crate2nix generate`, which a project reruns when its lockfile changes and commits, took \(.cpu | secs) CPU seconds and \(.wall | secs) wall at the \(.lock) lockfile." + "`crate2nix generate`, which a project reruns and commits when its lockfile changes, took " + + ($generate | map("\(.cpu | secs) CPU seconds at the \(.lock) lockfile") | join(" and ")) + "." ) ] | .[] diff --git a/bench/build/nix.nix b/bench/build/nix.nix index 5da8cfb1d..a747d26bc 100644 --- a/bench/build/nix.nix +++ b/bench/build/nix.nix @@ -10,17 +10,16 @@ mbxStore, }: let - # Only crane's variants evaluate moq's flake, which copies the whole checkout. - moq = builtins.getFlake "path:${src}"; system = builtins.currentSystem; - # moq's own nixpkgs and rust-overlay, from its lockfile. + # moq's own nixpkgs, rust-overlay, and crane, from its lockfile. lock = builtins.fromJSON (builtins.readFile "${src}/flake.lock"); - input = name: builtins.fetchTree lock.nodes.${lock.nodes.root.inputs.${name}}.locked; - pkgs = import (input "nixpkgs") { + locked = name: lock.nodes.${lock.nodes.root.inputs.${name}}.locked; + pkgs = import (builtins.fetchTree (locked "nixpkgs")) { inherit system; - overlays = [ (import (input "rust-overlay")) ]; + overlays = [ (import (builtins.fetchTree (locked "rust-overlay"))) ]; }; inherit (pkgs) lib; + crane' = with locked "crane"; builtins.getFlake "github:${owner}/${repo}/${rev}"; # The toolchain moq's overlay builds with, which pins the same channel. channel = (lib.importTOML "${src}/rust-toolchain.toml").toolchain.channel; @@ -37,20 +36,21 @@ let "force-frame-pointers=yes" ]; - # crane as moq builds it. - crane = lib.mapAttrs (name: _: moq.packages.${system}.${name}) packages; + # crane as moq builds it: its flake's overlay, keyed by cargo package. + moq = pkgs.extend (import "${src}/nix/overlay.nix" { crane = crane'; }); + crane = lib.mapAttrs (name: _: moq.${name}) packages; # crane with no dependency layer: every compilation goes through mbx, on a - # store outside the sandbox. `cargo` on PATH is mbx's shim. - mbxCargo = pkgs.runCommand "mbx-cargo" { } '' - mkdir -p $out/bin - ln -s ${builtins.storePath mbx}/libexec/mbx/cargo $out/bin/cargo - ''; + # store outside the sandbox. `cargo` on PATH is mbx's shim, which finds the + # real cargo after its own directory, so that directory goes on PATH as is. craneMbx = lib.mapAttrs ( _: drv: drv.overrideAttrs (old: { cargoArtifacts = null; - nativeBuildInputs = [ mbxCargo ] ++ old.nativeBuildInputs; + preConfigure = '' + export PATH=${builtins.storePath mbx}/libexec/mbx:$PATH + '' + + (old.preConfigure or ""); # mbx would move a target directory it finds into its store. CARGO_TARGET_DIR = "target"; MBX_CACHE_DIR = mbxStore; @@ -73,16 +73,16 @@ let cargo = toolchain; # Cargo's release default, which the other ways build with; buildRustCrate's is 1. defaultCodegenUnits = 16; - defaultCrateOverrides = pkgs.defaultCrateOverrides // overrides; }; - # Crates whose build scripts need more than nixpkgs' overrides give them. - overrides = { }; plan = pkgs.callPackage "${src}/Cargo.nix" { buildRustCrateForPkgs = - _: crate: buildRustCrate (crate // { extraRustcOpts = rustflags ++ (crate.extraRustcOpts or [ ]); }); + _: crate: + buildRustCrate (crate // { extraRustcOpts = rustflags ++ (crate.extraRustcOpts or [ ]); }); }; in - lib.mapAttrs (name: features: plan.workspaceMembers.${name}.build.override { inherit features; }) packages; + lib.mapAttrs ( + name: features: plan.workspaceMembers.${name}.build.override { inherit features; } + ) packages; in { inherit diff --git a/bench/build/nix.sh b/bench/build/nix.sh index 330f1aaf3..9eb637ec1 100755 --- a/bench/build/nix.sh +++ b/bench/build/nix.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash -# Measures moq's Nix release builds, built each way nix.nix offers, in CPU time. +# Measures moq's Nix release builds in CPU time, built each way nix.nix offers. # Usage and output are described in README.md. set -euo pipefail bench=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) root=$(cd "$bench/../.." && pwd) -# The build-bench shell carries hyperfine and crate2nix. +# The build-bench shell carries hyperfine, taplo, and cargo. if [[ -z ${RUST2_BENCH_NIGHTLY:-} ]]; then exec nix develop "$root#build-bench" --command "$bench/nix.sh" "$@" fi @@ -50,7 +50,7 @@ mapfile -t packages < <(field '.packages[]') mapfile -t bump < <(field '.bump[]') hub=$(field .hub) -# Private to this script, so a `just bench` running beside it keeps its checkout. +# Apart from run.sh's, so a `just bench` beside this run keeps its checkout. work=/var/tmp/rust2-bench-$(id -un)/nix results=$root/target/bench/nix out=${out:-$results/$(date -u +%Y%m%dT%H%M%SZ)} @@ -59,7 +59,7 @@ mkdir -p "$work" "$out/raw" "$out/logs" out=$(cd "$out" && pwd) src=$work/src -# As in run.sh: none of the caller's cargo or rustc settings reach the builds. +# As in run.sh: none of the caller's cargo, rustc, or mbx settings reach the builds. caller_vars=("${!CARGO@}" "${!RUSTC@}" "${!RUSTDOC@}" "${!RUSTUP@}" "${!MBX@}") for name in RUSTFLAGS CI TF_BUILD; do [[ -z ${!name+set} ]] || caller_vars+=("$name") @@ -75,10 +75,9 @@ PATH=$tools/bin:$PATH # mbx's own store path, which the Nix builds take as an input. mbx=$(dirname "$(dirname "$(readlink -f "$tools/bin/mbx")")") -# Every Nix variant builds in a store of its own under the work directory, so a -# run starts from nothing and nothing the host built is substituted. Builds run -# as this user, so their CPU time is this process's children's. Each runs on -# every core, with cargo's release codegen units. +# Each Nix variant builds in a store of its own, so it starts from nothing and +# substitutes nothing the host built. Builds run as this user, so their CPU time +# is this process's children's. store_args() { # variant printf '%s\n' --store "$work/$1/store" --option substituters https://cache.nixos.org \ --option max-jobs "$(getconf _NPROCESSORS_ONLN)" --option cores 0 \ @@ -121,14 +120,35 @@ checkout() { git -C "$src" clean -qfdx } -# Times one command, appending hyperfine's measurement to the step's record. +# Times one command with hyperfine, its output in the log. hyperfine runs a +# command without a shell, so one redirects it. time_cmd() { # json log command... - local json=$1 log=$2 + local json=$1 log=$2 cmd shift 2 - hyperfine --style none --runs 1 --output "$log" --export-json "$json" "$(printf '%q ' "$@")" >/dev/null || + cmd="$(printf '%q ' "$@") >$(printf '%q' "$log") 2>&1" + hyperfine --style none --runs 1 --export-json "$json" "bash -c $(printf '%q' "$cmd")" >/dev/null || { tail -60 "$log" >&2 && die "failed: $*"; } } +# One hyperfine measurement, as the results record it. +measurement() { # hyperfine.json + jq -c '.results[0].measurements[0] + | {wall: .time_wall_clock.value, cpu: .time_cpu.value, instructions: .instructions.value, rss: .memory_peak_resident.value}' "$1" +} + +# Appends one step's result. `compiled` counts the compilations that ran the +# compiler and `restored` those a cache served, each as the variant can tell. +record() { # variant scenario package derivations compiled restored eval build + jq -n -c --arg variant "$1" --arg scenario "$2" --arg package "$3" \ + --argjson derivations "$4" --argjson compiled "$5" --argjson restored "$6" --argjson eval "$7" --argjson build "$8" \ + '{$variant, $scenario, $package, $eval, $build, $derivations, $compiled, $restored}' >>"$out/results.jsonl" +} + +# The compilations an mbx build report says ran the compiler, and those it restored. +mbx_counts() { # report.json + jq -r '"\(.misses + .unconsulted) \(.hits)"' "$1" +} + # The derivations that compile Rust: crane's, and crate2nix's one per crate. measured() { # closure.json jq -r '.derivations | to_entries[] | select(.value.env.cargoVendorDir != null or .value.env.crateName != null) | .key' "$1" @@ -136,13 +156,14 @@ measured() { # closure.json # One step of a Nix variant: evaluate, realise what the build needs but does not # measure (toolchain, sources, vendored crates), then time the build. -nix_step() { # variant scenario attr - local variant=$1 scenario=$2 attr=$3 drv tag=$1.$2 +nix_step() { # variant scenario package attr + local variant=$1 scenario=$2 package=$3 attr=$4 drv output compiled restored=null tag=$1.$2 local -a nixo args measured prereqs built mapfile -t nixo < <(store_args "$variant") mapfile -t args < <(eval_args) time_cmd "$out/raw/$tag.eval.json" "$out/logs/$tag.eval.log" nix "${nixo[@]}" eval --raw "${args[@]}" "$attr.drvPath" - drv=$(nix "${nixo[@]}" eval --raw "${args[@]}" "$attr.drvPath") + drv=$(<"$out/logs/$tag.eval.log") + [[ $drv == /nix/store/*.drv ]] || die "$variant $scenario evaluated to $drv" nix "${nixo[@]}" derivation show -r "$drv" >"$work/closure.json" mapfile -t measured < <(measured "$work/closure.json") mapfile -t prereqs < <(jq -r --args ' @@ -150,60 +171,68 @@ nix_step() { # variant scenario attr | [$ARGS.positional[] | $d[.].inputs.drvs | to_entries[] | select(.key | IN($ARGS.positional[]) | not) | "/nix/store/\(.key)^\(.value.outputs | join(","))"] | unique[]' "${measured[@]}" <"$work/closure.json") - nix "${nixo[@]}" build --no-link "${prereqs[@]}" >"$out/logs/$tag.prereqs.log" 2>&1 || - { tail -40 "$out/logs/$tag.prereqs.log" >&2 && die "$variant failed to realise what $scenario needs"; } - # What the build will run, of the derivations that compile Rust. - mapfile -t built < <(nix "${nixo[@]}" build --dry-run --json "$drv^out" 2>&1 >/dev/null | + if ((${#prereqs[@]})); then + nix "${nixo[@]}" build --no-link "${prereqs[@]}" >"$out/logs/$tag.prereqs.log" 2>&1 || + { tail -40 "$out/logs/$tag.prereqs.log" >&2 && die "$variant failed to realise what $scenario needs"; } + fi + # The derivations that compile Rust which the build will run. + mapfile -t built < <(nix "${nixo[@]}" build --dry-run "$drv^out" 2>&1 | sed -n 's|^ /nix/store/\(.*\.drv\)$|\1|p' | grep -Fx -f <(printf '%s\n' "${measured[@]}") || true) time_cmd "$out/raw/$tag.build.json" "$out/logs/$tag.build.log" nix "${nixo[@]}" build --no-link -L "$drv^out" - jq -n -c --arg variant "$variant" --arg scenario "$scenario" --arg attr "$attr" \ - --argjson derivations "${#built[@]}" \ - --argjson compiled "$(grep -cE '^[^>]*> +(Compiling|Checking) |^[^>]*> Building ' "$out/logs/$tag.build.log" || true)" \ - --slurpfile eval "$out/raw/$tag.eval.json" --slurpfile build "$out/raw/$tag.build.json" \ - --slurpfile mbx <(if [[ $variant == crane-mbx ]]; then - cat "$(nix "${nixo[@]}" build --no-link --print-out-paths "$drv^out" | sed 's|^/|'"$work/$variant/store"'/|')/share/mbx/report.json" - fi) ' - def m: .[0].results[0] | {wall: .mean, cpu: (.user + .system), rss: .memory_usage_byte[0]}; - {$variant, $scenario, $attr, eval: ($eval | m), build: ($build | m), $derivations, $compiled, - mbx: ($mbx[0] // null | if . then {hits, misses} else null end)}' >>"$out/results.jsonl" + case $variant in + crane) compiled=$(grep -cE '^[^>]*> +(Compiling|Checking) ' "$out/logs/$tag.build.log" || true) ;; + crate2nix) compiled=${#built[@]} ;; + crane-mbx) + output=$(nix "${nixo[@]}" build --no-link --print-out-paths "$drv^out") + read -r compiled restored < <(mbx_counts "$work/$variant/store$output/share/mbx/report.json") + ;; + esac + record "$variant" "$scenario" "$package" "${#built[@]}" "$compiled" "$restored" \ + "$(measurement "$out/raw/$tag.eval.json")" "$(measurement "$out/raw/$tag.build.json")" } # One step of cargo with mbx: a fresh target directory, as a CI job starts with, # on the store earlier steps filled. cargo_step() { # scenario package - local scenario=$1 package=$2 tag=cargo-mbx.$1 - rm -rf "$CARGO_TARGET_DIR" + local scenario=$1 package=$2 tag=cargo-mbx.$1 compiled restored + rm -rf "$CARGO_TARGET_DIR" "$MBX_STATS_REPORT" ( cd "$src" time_cmd "$out/raw/$tag.build.json" "$out/logs/$tag.build.log" \ mbx build --release --locked --offline -p "$package" --features "$(jq -r --arg p "$package" '.[$p] | join(",")' <<<"$features")" ) - jq -n -c --arg scenario "$scenario" --arg attr "$package" \ - --argjson compiled "$(grep -cE '^ +Compiling ' "$out/logs/$tag.build.log" || true)" \ - --slurpfile build "$out/raw/$tag.build.json" --slurpfile mbx "$MBX_STATS_REPORT" ' - def m: .[0].results[0] | {wall: .mean, cpu: (.user + .system), rss: .memory_usage_byte[0]}; - {variant: "cargo-mbx", $scenario, $attr, eval: null, build: ($build | m), derivations: null, $compiled, - mbx: ($mbx[0] | {hits, misses})}' >>"$out/results.jsonl" + read -r compiled restored < <(mbx_counts "$MBX_STATS_REPORT") + record cargo-mbx "$scenario" "$package" null "$compiled" "$restored" null "$(measurement "$out/raw/$tag.build.json")" +} + +# crate2nix's build plan for the checkout. A project commits it, so its time is +# recorded apart from the builds. +generate() { + local lock tag + git -C "$src" diff --quiet -- Cargo.lock && lock=pinned || lock=bumped + tag=crate2nix.generate-$lock + (cd "$src" && time_cmd "$out/raw/$tag.json" "$out/logs/$tag.log" crate2nix generate) + measurement "$out/raw/$tag.json" | jq -c --arg lock "$lock" '{$lock} + .' >>"$out/generate.jsonl" } # Runs every scenario for one variant, from an empty store. run_variant() { # variant - local variant=$1 first=${packages[0]} second=${packages[1]} step + local variant=$1 first=${packages[0]} second=${packages[1]} attr echo "bench-nix: $variant" - chmod -R u+w "$work/$variant" 2>/dev/null || true + [[ ! -e $work/$variant ]] || chmod -R u+w "$work/$variant" rm -rf "${work:?}/$variant" mkdir -p "$work/$variant/mbx" checkout if [[ $variant == cargo-mbx ]]; then step() { cargo_step "$1" "$2"; } - # The toolchain, features, and flags of the Nix builds, as a CI job would set them. + # The toolchain, features, and flags of the Nix builds, set as a CI job would. export MBX_CACHE_DIR=$work/$variant/mbx MBX_GC_AUTO=0 MBX_STATS_REPORT=$work/$variant/report.json \ - CARGO_TARGET_DIR=$work/$variant/target PATH=$toolchain/bin:$mbx/bin:$PATH RUSTFLAGS=$rustflags CI=true + CARGO_TARGET_DIR=$work/$variant/target PATH=$toolchain/bin:$PATH RUSTFLAGS=$rustflags CI=true (cd "$src" && cargo fetch --locked -q) else nix copy --no-check-sigs --to "$work/$variant/store" "$mbx" - local attr=${variant/crane-mbx/craneMbx} - step() { nix_step "$variant" "$1" "$attr.$2"; } + attr=${variant/crane-mbx/craneMbx} + step() { nix_step "$variant" "$1" "$2" "$attr.$2"; } fi [[ $variant != crate2nix ]] || generate step clean "$first" @@ -215,17 +244,9 @@ run_variant() { # variant [[ $variant != crate2nix ]] || generate [[ $variant != cargo-mbx ]] || (cd "$src" && cargo fetch --locked -q) step lock-bump "$first" - unset -f step -} - -# crate2nix's build plan for the checkout. A project commits it, so its time is -# recorded but not measured as a build. -generate() { - local lock tag - git -C "$src" diff --quiet -- Cargo.lock && lock=pinned || lock=bumped - tag=crate2nix.generate-$lock - (cd "$src" && time_cmd "$out/raw/$tag.json" "$out/logs/$tag.log" crate2nix generate) - jq -c --arg lock "$lock" '.results[0] | {$lock, wall: .mean, cpu: (.user + .system)}' "$out/raw/$tag.json" >>"$out/generate.jsonl" + # A store holds gigabytes, more than a CI runner has free for four. + chmod -R u+w "$work/$variant" + rm -rf "${work:?}/$variant" } record_host @@ -235,7 +256,7 @@ toolchain=$(nix build --no-link --print-out-paths "${eval_args[@]}" toolchain) features=$(nix eval --json "${eval_args[@]}" packages) rustflags=$(nix eval --raw "${eval_args[@]}" rustflags --apply toString) for variant in "${variants[@]}"; do - # A subshell, so one variant's environment stays out of the next. + # A subshell keeps one variant's environment out of the next. (run_variant "$variant") done jq -n -r -f "$bench/nix-report.jq" --slurpfile host "$out/host.json" --slurpfile results "$out/results.jsonl" \ diff --git a/justfile b/justfile index 44d35af83..57d294040 100644 --- a/justfile +++ b/justfile @@ -20,7 +20,7 @@ check: shellcheck .claude/hooks/*.sh .codex/hooks/*.sh bench/*/*.sh shfmt -d .claude/hooks/*.sh .codex/hooks/*.sh bench/*/*.sh taplo format --check - nixfmt --check flake.nix + nixfmt --check flake.nix bench/build/nix.nix cargo fmt --all --check cargo clippy --locked --workspace --all-targets -- -D warnings just ts check @@ -50,7 +50,7 @@ bench-ffi *args: fix: shfmt -w .claude/hooks/*.sh .codex/hooks/*.sh bench/*/*.sh taplo format - nixfmt flake.nix + nixfmt flake.nix bench/build/nix.nix cargo fmt --all just ts fix just ffi fix From c10724dd1b28c132e86bafe3f82503e9e2e14016 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Sun, 25 Oct 2026 23:33:45 -0700 Subject: [PATCH 4/6] feat(bench): compare instructions, record the first Nix results, narrow the quest Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0115AT2ue6x9cT5e5J9SZtCn --- bench/build/README.md | 22 ++++++++++++++++++-- bench/build/nix-report.jq | 26 +++++++++++++---------- bench/build/nix.nix | 3 ++- quest/m1/README.md | 2 +- quest/m1/bench-nix.md | 44 ++++++++++++--------------------------- 5 files changed, 51 insertions(+), 46 deletions(-) diff --git a/bench/build/README.md b/bench/build/README.md index 5efb9c7a5..23ba0c30e 100644 --- a/bench/build/README.md +++ b/bench/build/README.md @@ -1,7 +1,7 @@ # Build-speed benchmark `just bench` measures Rust build time on the pinned projects and build configurations in [corpus.toml](corpus.toml). -It runs nightly in CI ([bench.yml](../../.github/workflows/bench.yml)), and on a PR that changes this harness, its workflow, the `build-bench` shell, or the `bench-ra-jit` package. +It runs nightly in CI ([bench.yml](../../.github/workflows/bench.yml)), and on a PR that changes this harness (apart from the [Nix comparison](#nix-builds)), its workflow, the `build-bench` shell, or the `bench-ra-jit` package. Every build-speed change here states its effect with its numbers. ```bash @@ -97,7 +97,25 @@ A cold edit or bump costs what `clean` does, so only the warm ones run. Each Nix build runs in a store of its own, which substitutes only from cache.nixos.org, so nothing the host or moq's cache built leaks in; before the timed build, it realises everything the build needs that does not compile Rust (toolchain, sources, vendored crates). That store runs its builds as this user in user namespaces, which makes their CPU time the `nix` process's own; Ubuntu reserves those namespaces for root unless `kernel.apparmor_restrict_unprivileged_userns` is 0. Evaluation is timed apart from the build, since crate2nix evaluates a derivation per crate. -crate2nix takes cargo's release codegen units rather than its default of one; `crate2nix generate`, which a project commits, is timed but not counted. +The report compares retired instructions, which barely move with machine load, and shows CPU time beside them. +crate2nix takes cargo's release codegen units rather than its default of one, so the variants differ in caching and not in codegen; `crate2nix generate`, which a project commits, is timed but not counted. + +### Nix builds for moq + +Measured on 2026-10-25 on a busy 32-CPU host, in billions of instructions, at the pin in nix.toml: + +| scenario | crane | crane-mbx | crate2nix | cargo-mbx | +| --- | ---: | ---: | ---: | ---: | +| `clean` | 3681 | 3377 | 4736 | 3378 | +| `second-package` | 5926 | 3945 | 5951 | 3924 | +| `edit-hub` | 1365 | 1351 | 1008 | 1276 | +| `lock-bump` | 3681 | 1123 | 672 | 1113 | +| total | 14756 | 9894 | 12461 | 9692 | + +- A lockfile bump costs crane a full dependency rebuild; the per-crate caches rebuild hyper and the 12 crates above it, a fifth of crane's cost or less. +- moq-cli shares 285 of its 548 compilations with moq-relay; the rest are crates or feature sets moq-relay does not build. crane shares none, and its dependency layer runs a `cargo check` of every dependency before building them. +- mbx restoring 434 compilations costs about as much as crate2nix rebuilding the edit outright. +- crate2nix's clean build costs 29% more, since it compiles build scripts and proc macros at opt-level 3 where cargo uses 0. With its own default of one codegen unit it fell to crane's cost (3695), and its edit and bump to 753 and 496: a lever for cargo too. ## Cranelift for moq's CI diff --git a/bench/build/nix-report.jq b/bench/build/nix-report.jq index 1241eb5d1..bf948d20a 100644 --- a/bench/build/nix-report.jq +++ b/bench/build/nix-report.jq @@ -4,7 +4,9 @@ def fixed(p): (. * pow(10; p) | round) / pow(10; p) | tostring; def secs: if . == null then "-" elif . < 10 then fixed(2) else fixed(1) end; +def giga: if . == null then "n/a" else (. / 1e9 | round | tostring) + "G" end; def pct: (. * 100 | fixed(1)) as $p | if . > 0 then "+\($p)%" else "\($p)%" end; +def vs($base): if $base == null or $base == 0 or . == null then "-" else . / $base - 1 | pct end; def count: if . == null then "-" else tostring end; def row: "| " + join(" | ") + " |"; @@ -18,23 +20,24 @@ $host[0] as $h "\($h.date) on \($h.cpu) (\($h.cpus) CPUs, \($h.memory)), \($h.os), runner `\($h.runner)`.", "rust2 `\($h.rust2[:12])`; moq `\($h.moq[:12])`; \($h.nix); \($h.mbx); crate2nix \($h.crate2nix).", "", - "One build per row, in CPU seconds: user plus system time of every process it ran.", + "One build per row: CPU seconds (user plus system time of every process it ran) and retired instructions.", + "Instructions barely move with machine load, so they are the comparison on a busy host; `vs crane` compares them.", "Each variant runs the scenarios in this order on one store, so every row after `clean` starts warm.", - "`vs crane` compares CPU time. Evaluation is timed apart, and the total adds it.", + "Evaluation is timed apart, and the totals add it.", "`derivations` counts the Nix derivations that compiled Rust; `compiled` counts rustc runs, as cargo's `Compiling` and `Checking` lines for crane, a derivation per crate for crate2nix, and mbx's misses for mbx; `restored` counts mbx's hits.", "", - (["scenario", "variant", "CPU", "vs crane", "wall", "eval CPU", "derivations", "compiled", "restored"] | row), - "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + (["scenario", "variant", "CPU", "instructions", "vs crane", "wall", "eval CPU", "derivations", "compiled", "restored"] | row), + "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", ( $scenarios[] as $s | $variants[] as $v | $by["\($s)/\($v)"] // empty - | $by["\($s)/crane"] as $base | [ $s, $v, (.build.cpu | secs), - (if $v == "crane" then "base" elif $base == null then "-" else .build.cpu / $base.build.cpu - 1 | pct end), + (.build.instructions | giga), + (if $v == "crane" then "base" else .build.instructions | vs($by["\($s)/crane"].build.instructions) end), (.build.wall | secs), (.eval.cpu | secs), (.derivations | count), @@ -44,13 +47,14 @@ $host[0] as $h | row ), "", - (["variant", "total CPU", "vs crane"] | row), - "| --- | ---: | ---: |", + (["variant", "total CPU", "total instructions", "vs crane"] | row), + "| --- | ---: | ---: | ---: |", ( - ($results | map(select(.variant == "crane") | .build.cpu + .eval.cpu) | add) as $base + def total($v; f): $results | map(select(.variant == $v) | (.build | f) + (.eval // {} | f // 0)) | add; + total("crane"; .instructions) as $base | $variants[] as $v - | ($results | map(select(.variant == $v) | .build.cpu + (.eval.cpu // 0)) | add) as $total - | [$v, ($total | secs), (if $v == "crane" then "base" elif $base == null then "-" else $total / $base - 1 | pct end)] + | [$v, (total($v; .cpu) | secs), (total($v; .instructions) | giga), + (if $v == "crane" then "base" else total($v; .instructions) | vs($base) end)] | row ), ( diff --git a/bench/build/nix.nix b/bench/build/nix.nix index a747d26bc..9b16daca7 100644 --- a/bench/build/nix.nix +++ b/bench/build/nix.nix @@ -71,7 +71,8 @@ let buildRustCrate = pkgs.buildRustCrate.override { rustc = toolchain; cargo = toolchain; - # Cargo's release default, which the other ways build with; buildRustCrate's is 1. + # Cargo's release default, which the other variants build with, so they + # compare caching and not codegen. buildRustCrate's own is 1. defaultCodegenUnits = 16; }; plan = pkgs.callPackage "${src}/Cargo.nix" { diff --git a/quest/m1/README.md b/quest/m1/README.md index c5ec15315..ca8191e98 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -19,4 +19,4 @@ are folded into the [exporter's](/quest/m1/ffi-export.md). The - [Waiting: publish the UniFFI fork](/quest/m1/uniffi-release.md) - the fork publishes to crates.io under `rust2-*` names, once the maintainer decides it should - [Exporter](/quest/m1/ffi-export.md) - moq's bindings exported from its crates' `pub` API, replacing the fork - [The exporter refuses what it can't export](/quest/m1/ffi-refuse.md) - `Option<&T>` parameters, unconstrained impl lifetimes, and unreachable objects work or are refused with a span -- [Nix builds in the build benchmark](/quest/m1/bench-nix.md) - moq's Nix release builds measured across crane, crane with mbx, crate2nix, and cargo with mbx, so moq can choose how they share compilations +- [Measure moq's fixed crane in the Nix benchmark](/quest/m1/bench-nix.md) - crane after moq's layering fixes beside the per-crate caches, so moq can choose how its Nix builds share compilations diff --git a/quest/m1/bench-nix.md b/quest/m1/bench-nix.md index d5593cbce..d40bdbb50 100644 --- a/quest/m1/bench-nix.md +++ b/quest/m1/bench-nix.md @@ -1,38 +1,20 @@ -# [M] Nix builds in the build benchmark +# [S] Measure moq's fixed crane in the Nix benchmark ## Goal -The [build-speed benchmark](/bench/build/README.md) measures moq's Nix release -builds (`nix build .#moq-relay` and a second package) in runner CPU time, so moq -can choose how they share compilations: - -- crane as moq builds it after its layering fixes; -- crane with mbx as its `RUSTC_WRAPPER`, on a local store mounted into the - sandbox with `extra-sandbox-paths`, as nixpkgs' ccache support does; -- crate2nix, one derivation per crate, shared through the Nix cache itself; -- cargo with mbx, as moq's CI builds today. - -The report says which one moq should adopt, as a follow-up quest in moq. +`just bench-nix` ([bench/build/README.md](/bench/build/README.md#nix-builds)) +measures moq's Nix release builds with crane as moq builds it after its layering +fixes, beside crane with mbx, crate2nix, and cargo with mbx, and the report says +which one moq should adopt, as a follow-up quest in moq. ## Plan -Decided with the maintainer on 2026-10-25: across moq's many CI and release -builds, each crate configuration compiles once and is reused, measured in runner -CPU time. Nix builds run sandboxed with no network, so they cannot reach mbx's -R2 store, and crane's two layers rebuild every dependency, or the whole -workspace, for changes that touch one crate. This bench decides between the -options rather than picking one up front. - -- Pin moq after its crane fixes (moq-dev/moq `quest/m0/ci/crane-layers.md`), - and rebuild moq's previous crane once from history for the before number. -- Scenarios, each with a cold and a warm cache: a clean build, a workspace edit - (the existing moq-net edit), a `Cargo.lock` bump of one dependency, and a - second package built after the first, which shows what the two share. -- Count what each option rebuilds as well as its time, and record the Nix - evaluation time separately, since crate2nix evaluates about 400 derivations. -- crate2nix reimplements cargo's build logic: note every crate that needs an - override (build scripts with `links`, proc macros, moq's kio `[patch]`). -- Nightly only, not on pull requests; the maintainer counts CPU time over wall - clock. +The harness and the other three variants landed, measured against moq's crane +before its fixes. Once moq-dev/moq `quest/m0/ci/crane-layers.md` merges, bump +nix.toml's pin to it, on its own since numbers on either side of a pin do not +compare, and compare the nightly numbers. -Public API: benchmark variants. +The first numbers bound what the fixes can win: a shared dependency layer can +only cut the second package's cost, while a lockfile bump still rebuilds every +dependency, three to five times what the per-crate caches pay. If moq adopts a +per-crate cache first, its crane fixes, and this quest, are moot. From 9c0b2618db9ce1db437e334c85ea8f3f0ae4f6af Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:27:21 -0700 Subject: [PATCH 5/6] chore(quest): finish bench-nix: moq explores crate2nix Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0115AT2ue6x9cT5e5J9SZtCn --- bench/build/README.md | 2 ++ quest/m1/README.md | 1 - quest/m1/bench-nix.md | 20 -------------------- 3 files changed, 2 insertions(+), 21 deletions(-) delete mode 100644 quest/m1/bench-nix.md diff --git a/bench/build/README.md b/bench/build/README.md index da306ebbd..cc0991edd 100644 --- a/bench/build/README.md +++ b/bench/build/README.md @@ -107,6 +107,8 @@ crate2nix takes cargo's release codegen units rather than its default of one, so ### Nix builds for moq +Decided on 2026-10-26: moq explores crate2nix, the cheapest on the edits and lockfile bumps CI repeats, and keeps its crane layers as they are. + Measured on 2026-10-25 on a busy 32-CPU host, in billions of instructions, at the pin in nix.toml: | scenario | crane | crane-mbx | crate2nix | cargo-mbx | diff --git a/quest/m1/README.md b/quest/m1/README.md index a69ac770b..4ea0f357f 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -18,6 +18,5 @@ are folded into the [exporter's](/quest/m1/ffi-export.md). The - [Waiting: publish the UniFFI fork](/quest/m1/uniffi-release.md) - the fork publishes to crates.io under `rust2-*` names, once the maintainer decides it should - [Exporter](/quest/m1/ffi-export.md) - moq's bindings exported from its crates' `pub` API, replacing the fork -- [Measure moq's fixed crane in the Nix benchmark](/quest/m1/bench-nix.md) - crane after moq's layering fixes beside the per-crate caches, so moq can choose how its Nix builds share compilations - [Land main through quest land](/quest/m1/land.md) - every queued PR lands as one tested batch through the shared gate - [Bump quest to quest land](/quest/m1/land-bump.md) - condition: kixelated/quest ships `quest land`; then bump the flake input diff --git a/quest/m1/bench-nix.md b/quest/m1/bench-nix.md deleted file mode 100644 index d40bdbb50..000000000 --- a/quest/m1/bench-nix.md +++ /dev/null @@ -1,20 +0,0 @@ -# [S] Measure moq's fixed crane in the Nix benchmark - -## Goal - -`just bench-nix` ([bench/build/README.md](/bench/build/README.md#nix-builds)) -measures moq's Nix release builds with crane as moq builds it after its layering -fixes, beside crane with mbx, crate2nix, and cargo with mbx, and the report says -which one moq should adopt, as a follow-up quest in moq. - -## Plan - -The harness and the other three variants landed, measured against moq's crane -before its fixes. Once moq-dev/moq `quest/m0/ci/crane-layers.md` merges, bump -nix.toml's pin to it, on its own since numbers on either side of a pin do not -compare, and compare the nightly numbers. - -The first numbers bound what the fixes can win: a shared dependency layer can -only cut the second package's cost, while a lockfile bump still rebuilds every -dependency, three to five times what the per-crate caches pay. If moq adopts a -per-crate cache first, its crane fixes, and this quest, are moot. From 50af5285c2d497a90c55d764117b5ed84cb14553 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:37:55 -0700 Subject: [PATCH 6/6] fix(bench): build crate2nix with moq's release profile and default features crate2nix's feature override replaced the packages' defaults, and it took 16 codegen units without LTO where moq's profile sets one unit and fat LTO. Totals now report unknown instructions as unknown, and private stores build in a directory of their own. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0115AT2ue6x9cT5e5J9SZtCn --- bench/build/README.md | 21 ++++++++++----------- bench/build/nix-report.jq | 4 +++- bench/build/nix.nix | 26 ++++++++++++++++++++------ bench/build/nix.sh | 7 ++++--- 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/bench/build/README.md b/bench/build/README.md index cc0991edd..02befadfe 100644 --- a/bench/build/README.md +++ b/bench/build/README.md @@ -103,26 +103,25 @@ Each Nix build runs in a store of its own, which substitutes only from cache.nix That store runs its builds as this user in user namespaces, which makes their CPU time the `nix` process's own; Ubuntu reserves those namespaces for root unless `kernel.apparmor_restrict_unprivileged_userns` is 0. Evaluation is timed apart from the build, since crate2nix evaluates a derivation per crate. The report compares retired instructions, which barely move with machine load, and shows CPU time beside them. -crate2nix takes cargo's release codegen units rather than its default of one, so the variants differ in caching and not in codegen; `crate2nix generate`, which a project commits, is timed but not counted. +crate2nix reads no Cargo profile, so nix.nix gives it moq's `[profile.release]`: one codegen unit, and fat LTO and symbol stripping on the linked packages. `crate2nix generate`, which a project commits, is timed but not counted. ### Nix builds for moq -Decided on 2026-10-26: moq explores crate2nix, the cheapest on the edits and lockfile bumps CI repeats, and keeps its crane layers as they are. +Decided on 2026-10-26: moq explores crate2nix, and keeps its crane layers as they are. -Measured on 2026-10-25 on a busy 32-CPU host, in billions of instructions, at the pin in nix.toml: +Measured on 2026-10-25 and 2026-10-26 on a busy 32-CPU host, in billions of instructions, at the pin in nix.toml: | scenario | crane | crane-mbx | crate2nix | cargo-mbx | | --- | ---: | ---: | ---: | ---: | -| `clean` | 3681 | 3377 | 4736 | 3378 | -| `second-package` | 5926 | 3945 | 5951 | 3924 | -| `edit-hub` | 1365 | 1351 | 1008 | 1276 | -| `lock-bump` | 3681 | 1123 | 672 | 1113 | -| total | 14756 | 9894 | 12461 | 9692 | +| `clean` | 3681 | 3377 | 4672 | 3378 | +| `second-package` | 5926 | 3945 | 6999 | 3924 | +| `edit-hub` | 1365 | 1351 | 1509 | 1276 | +| `lock-bump` | 3681 | 1123 | 1255 | 1113 | +| total, evaluation included | 14756 | 9894 | 14538 | 9692 | -- A lockfile bump costs crane a full dependency rebuild; the per-crate caches rebuild hyper and the 12 crates above it, a fifth of crane's cost or less. +- A lockfile bump costs crane a full dependency rebuild; the per-crate caches rebuild hyper and the 12 crates above it, about a third of crane's cost, most of it moq-relay's fat-LTO link. - moq-cli shares 285 of its 548 compilations with moq-relay; the rest are crates or feature sets moq-relay does not build. crane shares none, and its dependency layer runs a `cargo check` of every dependency before building them. -- mbx restoring 434 compilations costs about as much as crate2nix rebuilding the edit outright. -- crate2nix's clean build costs 29% more, since it compiles build scripts and proc macros at opt-level 3 where cargo uses 0. With its own default of one codegen unit it fell to crane's cost (3695), and its edit and bump to 753 and 496: a lever for cargo too. +- crate2nix costs 27% more than crane on a clean build and 18% more on the second package. Cargo compiles a dependency that only fat LTO reads to bitcode alone, and build scripts and proc macros at opt-level 0; buildRustCrate compiles every crate to machine code and bitcode at opt-level 3. ## Cranelift for moq's CI diff --git a/bench/build/nix-report.jq b/bench/build/nix-report.jq index bf948d20a..4821b564e 100644 --- a/bench/build/nix-report.jq +++ b/bench/build/nix-report.jq @@ -50,7 +50,9 @@ $host[0] as $h (["variant", "total CPU", "total instructions", "vs crane"] | row), "| --- | ---: | ---: | ---: |", ( - def total($v; f): $results | map(select(.variant == $v) | (.build | f) + (.eval // {} | f // 0)) | add; + # A variant with no evaluation adds none; a measurement a phase lacks leaves the total unknown. + def total($v; f): $results | map(select(.variant == $v) | [(.build | f), (if .eval == null then 0 else .eval | f end)]) + | flatten | if any(. == null) then null else add end; total("crane"; .instructions) as $base | $variants[] as $v | [$v, (total($v; .cpu) | secs), (total($v; .instructions) | giga), diff --git a/bench/build/nix.nix b/bench/build/nix.nix index 9b16daca7..5368857da 100644 --- a/bench/build/nix.nix +++ b/bench/build/nix.nix @@ -65,24 +65,38 @@ let }) ) crane; - # One derivation per crate, with crate2nix's generated build plan. + # One derivation per crate, with crate2nix's generated build plan. It reads no + # Cargo profile, so the packages take moq's `[profile.release]` here: + # buildRustCrate's one codegen unit already matches, and fat LTO and symbol + # stripping apply where cargo applies them, to the linked packages. crate2nix = let buildRustCrate = pkgs.buildRustCrate.override { rustc = toolchain; cargo = toolchain; - # Cargo's release default, which the other variants build with, so they - # compare caching and not codegen. buildRustCrate's own is 1. - defaultCodegenUnits = 16; }; + release = [ + "-C" + "lto=fat" + "-C" + "strip=symbols" + ]; plan = pkgs.callPackage "${src}/Cargo.nix" { buildRustCrateForPkgs = _: crate: - buildRustCrate (crate // { extraRustcOpts = rustflags ++ (crate.extraRustcOpts or [ ]); }); + buildRustCrate ( + crate + // { + extraRustcOpts = + rustflags ++ lib.optionals (packages ? ${crate.crateName}) release ++ (crate.extraRustcOpts or [ ]); + } + ); }; in + # Like cargo's `--features`, these add to the package's defaults. lib.mapAttrs ( - name: features: plan.workspaceMembers.${name}.build.override { inherit features; } + name: features: + plan.workspaceMembers.${name}.build.override { features = [ "default" ] ++ features; } ) packages; in { diff --git a/bench/build/nix.sh b/bench/build/nix.sh index 9eb637ec1..f6c8e383a 100755 --- a/bench/build/nix.sh +++ b/bench/build/nix.sh @@ -77,9 +77,10 @@ mbx=$(dirname "$(dirname "$(readlink -f "$tools/bin/mbx")")") # Each Nix variant builds in a store of its own, so it starts from nothing and # substitutes nothing the host built. Builds run as this user, so their CPU time -# is this process's children's. +# is this process's children's, in a build directory of the store's own, since +# the host's may belong to its daemon. store_args() { # variant - printf '%s\n' --store "$work/$1/store" --option substituters https://cache.nixos.org \ + printf '%s\n' --store "$work/$1/store" --option build-dir "$work/$1/build" --option substituters https://cache.nixos.org \ --option max-jobs "$(getconf _NPROCESSORS_ONLN)" --option cores 0 \ --option extra-sandbox-paths "/mbx=$work/$1/mbx" } @@ -221,7 +222,7 @@ run_variant() { # variant echo "bench-nix: $variant" [[ ! -e $work/$variant ]] || chmod -R u+w "$work/$variant" rm -rf "${work:?}/$variant" - mkdir -p "$work/$variant/mbx" + mkdir -p "$work/$variant/mbx" "$work/$variant/build" checkout if [[ $variant == cargo-mbx ]]; then step() { cargo_step "$1" "$2"; }