From beeea6f5f689d03224d987f9fe0b17a50231e716 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 09:18:40 -0800 Subject: [PATCH 1/2] chore(quest): claim quest/m0/nightly/lite06-join-hang.md Quest-Claim: 892abbc84463f3fd From 2930804bdb21010244f9a525cbd523eec532028f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 11:03:52 -0800 Subject: [PATCH 2/2] fix(bench): a paired base that hangs ends after ten silent minutes instead of holding the Nightly leg to its hour Co-Authored-By: Claude Sonnet 5.5 --- bench/README.md | 4 ++++ bench/build.slow.test.py | 17 ++++++++++++++-- bench/run.sh | 29 ++++++++++++++++++++++++++-- quest/m0/nightly/lite06-join-hang.md | 13 +++++++------ 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/bench/README.md b/bench/README.md index 3f7180c7f66..eb6449f3152 100644 --- a/bench/README.md +++ b/bench/README.md @@ -975,6 +975,10 @@ Each run's evidence directory keeps both builds' estimates (`criterion` and `paired.json`. The base build streams its diagnostics and retains them in `base-build.log`, including when it fails. +A target that prints nothing for ten minutes (`BENCH_CRITERION_IDLE_SECONDS`) is killed and +named as hung. A hung base only marks its target unavailable, so one hang in the revision +being compared cannot hold the leg to its hour and lose the current results. + Each revision builds in its own Cargo target directory (`base` and `current` under `CARGO_TARGET_DIR`, default `target/bench`). For an A/B comparison, use `run.sh`'s `BENCH_BASE` mode or a target directory per source tree: two trees diff --git a/bench/build.slow.test.py b/bench/build.slow.test.py index 138681ced9d..7c3cec4c307 100644 --- a/bench/build.slow.test.py +++ b/bench/build.slow.test.py @@ -15,7 +15,7 @@ class Builds(unittest.TestCase): def test_base_builds_report_errors_and_distinguish_new_targets(self): (ROOT / ".tmp").mkdir(exist_ok=True) - for base in ("broken", "existing", "new", "log-failure"): + for base in ("broken", "existing", "new", "log-failure", "hangs"): with self.subTest(base=base), tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory: repo = Path(directory) / "repo" repo.mkdir() @@ -39,9 +39,16 @@ def test_base_builds_report_errors_and_distinguish_new_targets(self): bench.write_text( 'compile_error!("base catalog source failed");\n' if base == "broken" + else 'fn main() { println!("base catalog ran"); loop { std::thread::park(); } }\n' + if base == "hangs" else 'fn main() { println!("base catalog ran"); }\n' ) - env = dict(os.environ, CARGO_TARGET_DIR=str(repo / "target"), BENCH_BASE="HEAD") + env = dict( + os.environ, + CARGO_TARGET_DIR=str(repo / "target"), + BENCH_BASE="HEAD", + BENCH_CRITERION_IDLE_SECONDS="2", + ) for key in ("BENCH_ARTIFACTS", "BENCH_RUNS", "RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER"): env.pop(key, None) @@ -91,6 +98,12 @@ def run(*args): report["unavailable"], [{"target": "moq-relay/catalog", "reason": "base build failed"}] ) self.assertEqual((evidence / "criterion-base.targets").read_text(), "") + elif base == "hangs": + # The silent base is killed and marked unavailable; the current run still reports. + self.assertIn("moq-relay/catalog hung: no output for 2s after: base catalog ran", result.stderr) + self.assertEqual( + report["unavailable"], [{"target": "moq-relay/catalog", "reason": "base run failed"}] + ) else: self.assertEqual(report["unavailable"], []) self.assertEqual("base catalog ran" in result.stdout, base in ("existing", "log-failure")) diff --git a/bench/run.sh b/bench/run.sh index c0e29d4c1f6..d4c988cc024 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -452,13 +452,38 @@ criterion_build_base() { criterion_build "$out" "$WORKTREE" ${args[@]+"${args[@]}"} } +# Seconds a target may print nothing before it counts as hung. Criterion prints at the start +# and end of every case, so the longest quiet stretch is one case's setup and measurement. +# Without the bound, a revision that hangs in one case holds the whole Nightly leg to its +# timeout, which cancels it and loses every other result. +CRITERION_IDLE_SECONDS=${BENCH_CRITERION_IDLE_SECONDS:-600} + # Run one built target as cargo bench would, from its package directory, with results under HOME/KEY. +# Kills it and fails when it prints nothing for CRITERION_IDLE_SECONDS. criterion_exec() { - local list=$1 home=$2 key=$3 package executable + local list=$1 home=$2 key=$3 package executable pid fd line last= shift 3 IFS=$'\t' read -r _ package executable < <(target_line "$list" "$key") || return 1 printf '\nCriterion %s: %s\n' "${home#"$RUN"/}" "$key" - (cd "$package" && CRITERION_HOME=$home/$key "$executable" --bench "$@" 128)); then + kill -KILL "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + exec {fd}<&- + printf 'Criterion %s hung: no output for %ds after: %s\n' "$key" "$CRITERION_IDLE_SECONDS" "$last" >&2 + return 1 + else + [[ -z $line ]] || printf '%s' "$line" + break + fi + done + exec {fd}<&- + wait "$pid" } target_line() { diff --git a/quest/m0/nightly/lite06-join-hang.md b/quest/m0/nightly/lite06-join-hang.md index 020c2859426..d29ae1f2bde 100644 --- a/quest/m0/nightly/lite06-join-hang.md +++ b/quest/m0/nightly/lite06-join-hang.md @@ -8,12 +8,13 @@ inside its hour with every cell, including the moq-lite-06 cells of ## Plan -The lite-06 join hang that kept those cells in their warm-up is fixed: a final -drop no longer hides a group the relay holds whole. Locally, every -`relay_session_join_*` cell finishes, the first lite-06 one in about 0.2 ms per -join. Confirm the first scheduled Nightly after the fix lands; never dispatch -Nightly. +The lite-06 join hang itself is fixed: a final drop no longer hides a group the +relay holds whole. The first scheduled Nightly after that fix still timed out +because its paired base, the previous Nightly's commit, predated the fix and hung +in the same cell; the base run now ends when it prints nothing for ten minutes and +only marks its target unavailable. Confirm the first scheduled Nightly after that +lands; never dispatch Nightly. ## Waiting -- moq-dev/moq: the first scheduled Nightly on `main` after the fix lands finishes its `Relay session benchmarks` job inside the hour; check with `gh run list --workflow nightly.yml --event schedule`, then delete this quest +- moq-dev/moq: the first scheduled Nightly on `main` after the idle bound lands finishes its `Relay session benchmarks` job inside the hour; check with `gh run list --workflow nightly.yml --event schedule`, then delete this quest