From c36783889ea8be005ce2e01a1f962c620007362a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 31 Oct 2026 16:14:42 -0700 Subject: [PATCH] ci: let the land gate stop just ci at the first failed module The host land gate ejects a failing batch whatever else fails, yet each failed batch on 2026-10-31 ran 20-50 more minutes after its first failure (check quest in 3s, check rs in 40s). CHECK_FAIL_FAST=1 stops a lane at its first failed module, cancels the running lanes, and starts no more. Without it, just ci still runs every lane to the end. Co-Authored-By: Claude Opus 5.5 --- justfile | 3 ++- sh/dispatch-query.test.py | 39 +++++++++++++++++++++++++++++++++++++++ sh/dispatch.sh | 15 ++++++++++++--- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/justfile b/justfile index c624e1380d6..9e2ce7d73d6 100644 --- a/justfile +++ b/justfile @@ -99,7 +99,8 @@ check $BASE="": # The landing worker runs this gate in one job. The test build also enforces # default-feature clippy; selected feature extremes run in the gate, while # the full feature matrix and optimized bindings run nightly and on release -# PRs. Missing tools fail under CI_STRICT. +# PRs. Missing tools fail under CI_STRICT. The host land gate also sets +# CHECK_FAIL_FAST=1, so a failing batch stops at its first failed module. # Run the landing worker's Linux gate over the branch diff against BASE. ci $BASE="": diff --git a/sh/dispatch-query.test.py b/sh/dispatch-query.test.py index ba72ce32f59..ea28b99998c 100644 --- a/sh/dispatch-query.test.py +++ b/sh/dispatch-query.test.py @@ -262,6 +262,45 @@ def test_failure_keeps_running_lanes_and_starts_remaining_modules(self): self.assertIn("obs compile", calls) self.assertIn("dart check", calls) + def test_fail_fast_cancels_running_lanes_and_starts_no_more(self): + # The land gate opts in: the parked lanes never finish on their own, so + # only the cancellation can end the run. + blocked = {} + for name, call in (("rs", '"rs test-changed "*'), ("cpp", '"cpp check"')): + started = self.root / f"{name}.started" + released = self.root / f"{name}.released" + os.mkfifo(started) + os.mkfifo(released) + blocked[name] = (call, started, released) + stub = "".join(f' {call}) echo >"{started}"; read -r <"{released}" ;;\n' + for call, started, released in blocked.values()) + reads = "; ".join(f'read -r <"{started}"' for _, started, _ in blocked.values()) + self.stub_ci(f'case "$*" in\n{stub} "gh test") {reads}; exit 3 ;;\nesac\n') + self.change("rs/moq-ffi/src/lib.rs") + self.change("js/net/src/index.ts") + result = subprocess.run(["bash", "sh/dispatch.sh", "ci", "origin/main"], cwd=self.repo, + env=dict(self.env, CHECK_FAIL_FAST="1"), text=True, capture_output=True, + timeout=60) + self.assertEqual(result.returncode, 3, result.stderr) + self.assertIn("ci: test gh failed", result.stderr) + self.assertRegex(result.stderr, r"\d+ test rs \(cancelled\)") + self.assertRegex(result.stderr, r"\d+ check cpp \(cancelled\)") + self.assertNotIn("ci: test rs failed", result.stderr) + calls = (self.root / "calls").read_text().splitlines() + self.assertNotIn("go check", calls) + self.assertNotIn("obs compile", calls) + + def test_fail_fast_stops_a_serial_run_at_the_first_failure(self): + self.stub_ci('case "$*" in "js check-changed "*) exit 3 ;; "gh check") exit 4 ;; esac\n') + self.change("js/net/src/index.ts") + self.change("dart/example.dart") + result = subprocess.run(["bash", "sh/dispatch.sh", "ci", "origin/main"], cwd=self.repo, + env=dict(self.env, CHECK_JOBS="1", CHECK_FAIL_FAST="1"), text=True, + capture_output=True) + self.assertNotEqual(result.returncode, 0, result.stderr) + failures = [line for line in result.stderr.splitlines() if line.startswith("ci: ") and line.endswith(" failed")] + self.assertEqual(len(failures), 1, result.stderr) + def test_failure_continues_independent_modules_in_the_same_lane(self): for jobs in ("1", "3"): with self.subTest(jobs=jobs): diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 2686cb8d571..19563fd9af4 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -4,6 +4,8 @@ # them in order and streams their output. CHECK_MODULES limits the selection to # the modules it names, or skips those named with a leading `-`, so a gate too # large for one job splits across several. +# CHECK_FAIL_FAST=1 stops every lane at the first failed module, for the land +# gate, which ejects a failing batch either way and gains nothing from the rest. # `--query` prints the selection before CI installs its toolchain and caches. # `ci-platform` exists only as a query for the native platform workflows. set -euo pipefail @@ -452,6 +454,7 @@ run_lane() { record failed ((lane_status)) || lane_status=$status current= + [[ -z "${CHECK_FAIL_FAST:-}" ]] || return "$lane_status" break fi record @@ -522,10 +525,12 @@ trap 'exit 143' TERM # run's stderr as it starts and ends, so a run cut off mid-phase, such as by a # job timeout that never reaches this script, names the phase it stopped in. # Every lane completes even when another fails, so one unavailable check cannot -# hide unrelated failures. +# hide unrelated failures; under CHECK_FAIL_FAST the first failure cancels the +# running lanes and starts no more. set -m +stopping= while true; do - while [[ -z "$lone" ]] && ((next < ${#lanes[@]} && ${#running[@]} < limit)); do + while [[ -z "$lone$stopping" ]] && ((next < ${#lanes[@]} && ${#running[@]} < limit)); do name=${lanes[next]} if [[ -v "alone[$name]" ]] && ((${#running[@]})); then break @@ -547,9 +552,13 @@ while true; do name=${running[$pid]} unset "running[$pid]" [[ "$pid" != "$lone" ]] || lone= - if ((status)); then + if ((status)) && [[ -z "$stopping" ]]; then failed+=("$name") ((failed_status)) || failed_status=$status + if [[ -n "${CHECK_FAIL_FAST:-}" ]]; then + stopping=1 + cancel + fi fi ((limit == 1)) || cat "$logs/$name.log" done