Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -975,6 +975,10 @@ Each run's evidence directory keeps both builds' estimates (`criterion` and
`paired.json`. The base build streams its diagnostics and retains them in
`base-build.log`, including when it fails.

A target that prints nothing for ten minutes (`BENCH_CRITERION_IDLE_SECONDS`) is killed and
named as hung. A hung base only marks its target unavailable, so one hang in the revision
being compared cannot hold the leg to its hour and lose the current results.

Each revision builds in its own Cargo target directory (`base` and `current`
under `CARGO_TARGET_DIR`, default `target/bench`). For an A/B comparison, use
`run.sh`'s `BENCH_BASE` mode or a target directory per source tree: two trees
Expand Down
17 changes: 15 additions & 2 deletions bench/build.slow.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
class Builds(unittest.TestCase):
def test_base_builds_report_errors_and_distinguish_new_targets(self):
(ROOT / ".tmp").mkdir(exist_ok=True)
for base in ("broken", "existing", "new", "log-failure"):
for base in ("broken", "existing", "new", "log-failure", "hangs"):
with self.subTest(base=base), tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory:
repo = Path(directory) / "repo"
repo.mkdir()
Expand All @@ -39,9 +39,16 @@ def test_base_builds_report_errors_and_distinguish_new_targets(self):
bench.write_text(
'compile_error!("base catalog source failed");\n'
if base == "broken"
else 'fn main() { println!("base catalog ran"); loop { std::thread::park(); } }\n'
if base == "hangs"
else 'fn main() { println!("base catalog ran"); }\n'
)
env = dict(os.environ, CARGO_TARGET_DIR=str(repo / "target"), BENCH_BASE="HEAD")
env = dict(
os.environ,
CARGO_TARGET_DIR=str(repo / "target"),
BENCH_BASE="HEAD",
BENCH_CRITERION_IDLE_SECONDS="2",
)
for key in ("BENCH_ARTIFACTS", "BENCH_RUNS", "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER"):
env.pop(key, None)

Expand Down Expand Up @@ -91,6 +98,12 @@ def run(*args):
report["unavailable"], [{"target": "moq-relay/catalog", "reason": "base build failed"}]
)
self.assertEqual((evidence / "criterion-base.targets").read_text(), "")
elif base == "hangs":
# The silent base is killed and marked unavailable; the current run still reports.
self.assertIn("moq-relay/catalog hung: no output for 2s after: base catalog ran", result.stderr)
self.assertEqual(
report["unavailable"], [{"target": "moq-relay/catalog", "reason": "base run failed"}]
)
else:
self.assertEqual(report["unavailable"], [])
self.assertEqual("base catalog ran" in result.stdout, base in ("existing", "log-failure"))
Expand Down
29 changes: 27 additions & 2 deletions bench/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -452,13 +452,38 @@ criterion_build_base() {
criterion_build "$out" "$WORKTREE" ${args[@]+"${args[@]}"}
}

# Seconds a target may print nothing before it counts as hung. Criterion prints at the start
# and end of every case, so the longest quiet stretch is one case's setup and measurement.
# Without the bound, a revision that hangs in one case holds the whole Nightly leg to its
# timeout, which cancels it and loses every other result.
CRITERION_IDLE_SECONDS=${BENCH_CRITERION_IDLE_SECONDS:-600}

# Run one built target as cargo bench would, from its package directory, with results under HOME/KEY.
# Kills it and fails when it prints nothing for CRITERION_IDLE_SECONDS.
criterion_exec() {
local list=$1 home=$2 key=$3 package executable
local list=$1 home=$2 key=$3 package executable pid fd line last=
shift 3
IFS=$'\t' read -r _ package executable < <(target_line "$list" "$key") || return 1
printf '\nCriterion %s: %s\n' "${home#"$RUN"/}" "$key"
(cd "$package" && CRITERION_HOME=$home/$key "$executable" --bench "$@" </dev/null)
exec {fd}< <(cd "$package" && CRITERION_HOME=$home/$key exec "$executable" --bench "$@" </dev/null)
pid=$!
while true; do
if IFS= read -r -t "$CRITERION_IDLE_SECONDS" -u "$fd" line; then
printf '%s\n' "$line"
[[ -z $line ]] || last=$line
elif (($? > 128)); then
kill -KILL "$pid" 2>/dev/null || true
wait "$pid" 2>/dev/null || true
exec {fd}<&-
printf 'Criterion %s hung: no output for %ds after: %s\n' "$key" "$CRITERION_IDLE_SECONDS" "$last" >&2
return 1
else
[[ -z $line ]] || printf '%s' "$line"
break
fi
done
exec {fd}<&-
wait "$pid"
}

target_line() {
Expand Down
13 changes: 7 additions & 6 deletions quest/m0/nightly/lite06-join-hang.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,13 @@ inside its hour with every cell, including the moq-lite-06 cells of

## Plan

The lite-06 join hang that kept those cells in their warm-up is fixed: a final
drop no longer hides a group the relay holds whole. Locally, every
`relay_session_join_*` cell finishes, the first lite-06 one in about 0.2 ms per
join. Confirm the first scheduled Nightly after the fix lands; never dispatch
Nightly.
The lite-06 join hang itself is fixed: a final drop no longer hides a group the
relay holds whole. The first scheduled Nightly after that fix still timed out
because its paired base, the previous Nightly's commit, predated the fix and hung
in the same cell; the base run now ends when it prints nothing for ten minutes and
only marks its target unavailable. Confirm the first scheduled Nightly after that
lands; never dispatch Nightly.

## Waiting

- moq-dev/moq: the first scheduled Nightly on `main` after the fix lands finishes its `Relay session benchmarks` job inside the hour; check with `gh run list --workflow nightly.yml --event schedule`, then delete this quest
- moq-dev/moq: the first scheduled Nightly on `main` after the idle bound lands finishes its `Relay session benchmarks` job inside the hour; check with `gh run list --workflow nightly.yml --event schedule`, then delete this quest