Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -251,12 +251,17 @@ jobs:
access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }}
secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }}

# Inside the job's hour, so a workload that overruns fails the job, where the
# failure-based triage sees it, instead of cancelling it, and later steps still run.
- name: Run workload
id: workload
timeout-minutes: 55
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
run: nix develop ${{ matrix.shell }} --command just ${{ matrix.recipe }}
env:
NEXTEST_PROFILE: default
BENCH_BASE: ${{ matrix.paired && (inputs.bench-base || 'previous-nightly') || '' }}
# The previous Nightly's copy is the paired base's executables (see sh/gh/nightly.sh).
BENCH_BUILDS_ARTIFACT: ${{ matrix.paired && format('builds-{0}', matrix.artifact) || '' }}
GH_TOKEN: ${{ matrix.paired && github.token || '' }}

- name: Record benchmark outcome
Expand All @@ -271,6 +276,16 @@ jobs:
path: ${{ matrix.artifact-path }}
retention-days: 45

# Only scheduled runs serve as a later run's base.
- name: Keep bench executables
if: ${{ matrix.paired && github.event_name == 'schedule' && !cancelled() }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: builds-${{ matrix.artifact }}
path: target/bench/builds
if-no-files-found: ignore
retention-days: 3

# After the upload, so the benchmark site still records tonight's history and
# publishes the comparison that failed this job before the alert links to it.
- name: Fail on paired regressions
Expand Down
11 changes: 8 additions & 3 deletions bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -924,9 +924,14 @@ the table requires no new renderer or data branch. A source's `paired.json`, as
### Paired comparison

Each Nightly's Rust bench groups (`rust-bench`, `rust-shared`, `rust-relay`,
`rust-extra` and `rust-uring`) also build the previous scheduled Nightly's commit beside their
own, on the same runner, so night-to-night runner variance drops out. Each
built target runs once per side in the trend's quick mode, alternating which
`rust-extra` and `rust-uring`) also run the previous scheduled Nightly's commit beside their
own, on the same runner, so night-to-night runner variance drops out. That
night kept its bench executables as an artifact, which serve as the base
instead of a rebuild: the Rust cache cannot hold a linked executable, and
relinking `rust-bench`'s 32 fat-LTO targets took seven of its sixty minutes.
When that night kept no executable for some target, or one cannot load on
this runner, as after a library bump, the base is built. Each
target runs once per side in the trend's quick mode, alternating which
side goes first; the current side is the trend point. A target whose base
executable is byte-identical to the current one is not compared: it is listed
as the same build, since only noise could tell them apart. A case more than 10%
Expand Down
69 changes: 60 additions & 9 deletions bench/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,49 @@ unavailable() {
printf 'Paired comparison unavailable for %s: %s\n' "$1" "$2" >&2
}

# Copy the executables a list names to BENCH_KEEP_BUILDS, with the commit they were built
# from, so a later run can measure them as its base without relinking them.
keep_builds() {
local list=$1 dir=$BENCH_KEEP_BUILDS index=0 key package executable
rm -rf "$dir"
mkdir -p "$dir/bin"
(cd "$ROOT" && git rev-parse HEAD) >"$dir/commit"
: >"$dir/targets.tsv"
while IFS=$'\t' read -r key package executable; do
cp "$executable" "$dir/bin/$index"
printf '%s\t%s\t%s\n' "$key" "${package#"$ROOT"/}" "bin/$index" >>"$dir/targets.tsv"
index=$((index + 1))
done <"$list"
}

# List the base's kept executables in OUT, as criterion_build would, from the base checkout's
# packages. Fails when BENCH_BASE_BUILDS holds no builds of BASE covering every current target
# that can load on this host.
kept_base() {
local out=$1 current=$2 dir=${BENCH_BASE_BUILDS:-} key package file loaded
[[ -n $dir && -s $dir/targets.tsv && -f $dir/commit && $(<"$dir/commit") == "$BASE" ]] || return 1
dir=$(cd "$dir" && pwd -P)
while IFS=$'\t' read -r key _; do
[[ -n $(target_line "$dir/targets.tsv" "$key") ]] || return 1
done <"$current"
: >"$out"
while IFS=$'\t' read -r key package file; do
# An artifact download drops the executable bit.
chmod +x "$dir/$file"
# The executables link against the store paths of the shell that built them, which a
# fresh runner lacks after a toolchain or library bump. The loader alone, never main,
# resolves the interpreter and every library: anything missing builds the base instead.
if ! loaded=$(LD_TRACE_LOADED_OBJECTS=1 "$dir/$file" 2>&1); then
printf 'Kept base executable %s has no loader here\n' "$key" >&2
return 1
elif [[ $loaded == *"not found"* ]]; then
printf 'Kept base executable %s lacks a library here:\n%s\n' "$key" "$loaded" >&2
return 1
fi
printf '%s\t%s\t%s\n' "$key" "$WORKTREE/$package" "$dir/$file" >>"$out"
done <"$dir/targets.tsv"
}

# Measure the selected targets in quick mode. With a base, run each target's base build
# beside it, alternating which goes first, then confirm the screen's worst slowdowns.
# A failed current target fails the run; the base only marks its targets unavailable.
Expand All @@ -506,6 +549,7 @@ measure_criterion() {
local key side targets index=0 status=0
local filters=(${CRITERION_FILTER_ARGS[@]+"${CRITERION_FILTER_ARGS[@]}"})
criterion_build "$current" "$ROOT" ${CRITERION_CARGO_ARGS[@]+"${CRITERION_CARGO_ARGS[@]}"}
if [[ -n ${BENCH_KEEP_BUILDS:-} ]]; then keep_builds "$current"; fi
mapfile -t targets < <(cut -f1 "$current")
if ((${#targets[@]} == 0)); then
printf 'no Criterion bench targets selected\n' >&2
Expand All @@ -517,15 +561,22 @@ measure_criterion() {
mkdir -p "$RUN/base-context"
(cd "$WORKTREE" && python3 "$ROOT/bench/metadata.py" "$RUN/base-context" "$BASE" comparison-base)
printf 'Criterion paired with base %s\n' "$BASE"
printf 'Base build log: %s/base-build.log\n' "$RUN"
local build_status=(0 0)
criterion_build_base "$base" 2>&1 | tee "$RUN/base-build.log" >&2 || build_status=("${PIPESTATUS[@]}")
if ((build_status[1] != 0)); then
printf 'Failed to write base build log: %s/base-build.log\n' "$RUN" >&2
fi
if ((build_status[0] != 0)); then
: >"$base"
for key in "${targets[@]}"; do unavailable "$key" "base build failed"; done
if kept_base "$base" "$current"; then
printf 'Base executables kept by an earlier run: %s\n' "$BENCH_BASE_BUILDS"
else
if [[ -n ${BENCH_BASE_BUILDS:-} ]]; then
printf 'No kept base executable of every target in %s; building the base\n' "$BENCH_BASE_BUILDS"
fi
printf 'Base build log: %s/base-build.log\n' "$RUN"
local build_status=(0 0)
criterion_build_base "$base" 2>&1 | tee "$RUN/base-build.log" >&2 || build_status=("${PIPESTATUS[@]}")
if ((build_status[1] != 0)); then
printf 'Failed to write base build log: %s/base-build.log\n' "$RUN" >&2
fi
if ((build_status[0] != 0)); then
: >"$base"
for key in "${targets[@]}"; do unavailable "$key" "base build failed"; done
fi
fi
fi
: >"$RUN/paired-identical.txt"
Expand Down
2 changes: 2 additions & 0 deletions js/crypto/bench/pump.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,10 @@ for (const bytes of [160, 1_160, 16_384, 262_144]) {
// The first round warms the JIT and key imports and is discarded.
for (let round = 0; round < 6; round++) {
const generation = credential.produce();
// No publisher limit: every group is sealed before the first is read.
const net = new Broadcast.Producer().createTrack(await generation.name("video"), {
timescale: Time.Timescale.MILLI,
maxAge: null,
});
const consumer = credential
.consume(generation.epoch)
Expand Down
14 changes: 11 additions & 3 deletions js/json/bench/stream.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
/** Sweep buffered JSON stream reads over record count, payload size, and track compression. */
import { Track } from "@moq/net";
import { Time, Track } from "@moq/net";
import { Consumer, Producer } from "../src/stream/index.ts";

// A scale-less log opens its one group at this key, and every record carries it.
const at = Time.Timestamp.unscaled(0n);

console.log("records,payload_bytes,compress,ns_per_record");
for (const records of [64, 1024]) {
for (const bytes of [16, 1024]) {
Expand All @@ -11,14 +14,19 @@ for (const records of [64, 1024]) {
const track = new Track.Producer("stream-bench").accept({ compression: compress ? "deflate" : "none" });
const producer = new Producer<{ n: number; text: string }>({ track });
const text = "x".repeat(bytes);
for (let n = 0; n < records; n++) producer.append({ value: { n, text } });
for (let n = 0; n < records; n++) producer.append({ value: { n, text }, at });
producer.finish();

const consumer = new Consumer<{ n: number; text: string }>({ track: track.subscribe() });
const start = performance.now();
for (let n = 0; n < records; n++) {
const record = await consumer.next();
if (record?.value.n !== n || record.value.text !== text || record.at !== undefined) {
if (
record?.value.n !== n ||
record.value.text !== text ||
record.at.scale !== undefined ||
record.at.value !== at.value
) {
throw new Error("stream benchmark lost a record");
}
}
Expand Down
15 changes: 6 additions & 9 deletions js/net/bench/requests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,7 @@ for (const touched of ["watched", "unrelated"])
for (const followers of [1, 8, 32]) {
const origin = new Producer();
const path = Path.from("room/0");
const broadcasts = Array.from({ length: routes }, (_, i) => {
const broadcastRoute = origin.route(Path.from(`room/${i}`));
const broadcast = broadcastRoute.broadcast;
return broadcast;
});
const broadcasts = Array.from({ length: routes }, (_, i) => origin.route(Path.from(`room/${i}`)));
const requests = Array.from({ length: followers }, () => origin.follow(path));
await settle();
let current = broadcasts[0];
Expand All @@ -40,8 +36,7 @@ for (const touched of ["watched", "unrelated"])
for (let round = 0; round < 6; round++) {
const start = performance.now();
for (let i = 0; i < 128; i++) {
const nextRoute = origin.route(touched === "watched" ? path : Path.from("other"));
const next = nextRoute.broadcast;
const next = origin.route(touched === "watched" ? path : Path.from("other"));
if (touched === "watched") {
current.close();
current = next;
Expand All @@ -50,8 +45,9 @@ for (const touched of ["watched", "unrelated"])
unrelated = next;
}
await settle();
// A follower reads its own consumer of the route's broadcast, so match the instance.
for (const request of requests)
if (request.active.peek()?.closed !== current.closed)
if (request.active.peek()?.instance !== current.instance)
throw new Error("wrong publisher instance");
}
if (round > 0) samples.push(((performance.now() - start) * 1000) / 128);
Expand Down Expand Up @@ -142,7 +138,8 @@ for (const failovers of [0, 1, 16, 128])
const primary = origin.claim(prefix, { instance: { id: instance, epoch: 0n }, cost: 0n });
const ar = primary.requested();
const br = origin.claim(prefix, { instance: { id: instance, epoch: 0n }, cost: 1n }).requested();
const held = origin.request(path);
// Pinned, or the path's earlier instance would answer it and this claim never hears it.
const held = origin.request(path, { instance });
const source = new Broadcast();
sources.push(source);
const answer = (await ar.next()).value;
Expand Down
4 changes: 3 additions & 1 deletion quest/m0/nightly/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ children for the causes nothing owned.

This README's own work, after the children:

- Confirm a scheduled run on `main` passes; never dispatch Nightly. Failures
- Confirm a scheduled run on `main` passes; never dispatch Nightly. A
triage counts a cancelled job as failed: a workload that overruns fails at
its step's timeout, so a job cancelled at its own timeout hung outside it. Failures
already owned elsewhere stay there: the audio near-zero p95 row belongs to
[audio playout](/quest/m2/audio-playout/README.md), the
macOS `receive_credit` timeout has no owner since its quest was deleted in
Expand Down
24 changes: 0 additions & 24 deletions quest/m0/nightly/bench-timeouts.md

This file was deleted.

24 changes: 22 additions & 2 deletions sh/gh/nightly.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,25 @@ esac

# BENCH_BASE pairs every Rust bench target with its build at that revision, run on the
# same machine; `previous-nightly` is the commit of the latest other scheduled Nightly on main.
# That run kept its bench executables as the BENCH_BUILDS_ARTIFACT artifact. The cache
# can't hold a linked executable, so reusing them saves relinking every target at the base.
base_builds=
if [[ -n ${BENCH_BASE:-} ]]; then
if [[ $report != rust ]]; then
echo "nightly: BENCH_BASE applies only to the Rust bench groups" >&2
exit 2
fi
if [[ $BENCH_BASE == previous-nightly ]]; then
BENCH_BASE=$(bash "$(dirname "${BASH_SOURCE[0]}")/previous-nightly.sh")
read -r base_run BENCH_BASE < <(bash "$(dirname "${BASH_SOURCE[0]}")/previous-nightly.sh" --run) || exit 1
if [[ -n ${BENCH_BUILDS_ARTIFACT:-} ]]; then
base_builds=$(git rev-parse --show-toplevel)/target/bench/base-builds/$1
rm -rf "$base_builds"
# A previous leg that ended before its builds kept none; the base is built instead.
if ! gh run download "$base_run" --name "$BENCH_BUILDS_ARTIFACT" --dir "$base_builds"; then
echo "nightly: run $base_run kept no $BENCH_BUILDS_ARTIFACT; building the base" >&2
base_builds=
fi
fi
fi
# A shallow CI checkout has only its own commit. Fetching a branch name only updates
# FETCH_HEAD, so the base is exported as the commit it resolved to.
Expand All @@ -39,9 +51,12 @@ if [[ -n ${BENCH_BASE:-} ]]; then
export BENCH_BASE=$base
fi

group=$1
if [[ -n $report ]]; then
root=$(git rev-parse --show-toplevel)
out=${BENCH_RESULTS:-$root/target/bench/nightly/$1}
builds=$root/target/bench/builds/$group
rm -rf "$builds"
mkdir -p "$out"
export BENCH_RUNS="$out/criterion-runs.txt"
: >"$BENCH_RUNS"
Expand All @@ -52,10 +67,15 @@ fi
# shellcheck source=sh/checks.sh
source "$(dirname "${BASH_SOURCE[0]}")/../checks.sh"

# Each label keeps its executables for the next Nightly's base, under the directory
# the workflow uploads.
criterion() {
local label=$1
shift
run env BENCH_LABEL="$label" BENCH_ARTIFACTS="$out/runs" just gh criterion "$@"
run env BENCH_LABEL="$label" BENCH_ARTIFACTS="$out/runs" \
BENCH_KEEP_BUILDS="$builds/$label" \
BENCH_BASE_BUILDS="${base_builds:+$base_builds/$group/$label}" \
just gh criterion "$@"
}

javascript() {
Expand Down
Loading