Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion justfile
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,8 @@ check $BASE="":
# The landing worker runs this gate in one job. The test build also enforces
# default-feature clippy; selected feature extremes run in the gate, while
# the full feature matrix and optimized bindings run nightly and on release
# PRs. Missing tools fail under CI_STRICT.
# PRs. Missing tools fail under CI_STRICT. The host land gate also sets
# CHECK_FAIL_FAST=1, so a failing batch stops at its first failed module.

# Run the landing worker's Linux gate over the branch diff against BASE.
ci $BASE="":
Expand Down
39 changes: 39 additions & 0 deletions sh/dispatch-query.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,45 @@ def test_failure_keeps_running_lanes_and_starts_remaining_modules(self):
self.assertIn("obs compile", calls)
self.assertIn("dart check", calls)

def test_fail_fast_cancels_running_lanes_and_starts_no_more(self):
# The land gate opts in: the parked lanes never finish on their own, so
# only the cancellation can end the run.
blocked = {}
for name, call in (("rs", '"rs test-changed "*'), ("cpp", '"cpp check"')):
started = self.root / f"{name}.started"
released = self.root / f"{name}.released"
os.mkfifo(started)
os.mkfifo(released)
blocked[name] = (call, started, released)
stub = "".join(f' {call}) echo >"{started}"; read -r <"{released}" ;;\n'
for call, started, released in blocked.values())
reads = "; ".join(f'read -r <"{started}"' for _, started, _ in blocked.values())
self.stub_ci(f'case "$*" in\n{stub} "gh test") {reads}; exit 3 ;;\nesac\n')
self.change("rs/moq-ffi/src/lib.rs")
self.change("js/net/src/index.ts")
result = subprocess.run(["bash", "sh/dispatch.sh", "ci", "origin/main"], cwd=self.repo,
env=dict(self.env, CHECK_FAIL_FAST="1"), text=True, capture_output=True,
timeout=60)
self.assertEqual(result.returncode, 3, result.stderr)
self.assertIn("ci: test gh failed", result.stderr)
self.assertRegex(result.stderr, r"\d+ test rs \(cancelled\)")
self.assertRegex(result.stderr, r"\d+ check cpp \(cancelled\)")
self.assertNotIn("ci: test rs failed", result.stderr)
calls = (self.root / "calls").read_text().splitlines()
self.assertNotIn("go check", calls)
self.assertNotIn("obs compile", calls)

def test_fail_fast_stops_a_serial_run_at_the_first_failure(self):
self.stub_ci('case "$*" in "js check-changed "*) exit 3 ;; "gh check") exit 4 ;; esac\n')
self.change("js/net/src/index.ts")
self.change("dart/example.dart")
result = subprocess.run(["bash", "sh/dispatch.sh", "ci", "origin/main"], cwd=self.repo,
env=dict(self.env, CHECK_JOBS="1", CHECK_FAIL_FAST="1"), text=True,
capture_output=True)
self.assertNotEqual(result.returncode, 0, result.stderr)
failures = [line for line in result.stderr.splitlines() if line.startswith("ci: ") and line.endswith(" failed")]
self.assertEqual(len(failures), 1, result.stderr)

def test_failure_continues_independent_modules_in_the_same_lane(self):
for jobs in ("1", "3"):
with self.subTest(jobs=jobs):
Expand Down
15 changes: 12 additions & 3 deletions sh/dispatch.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
# them in order and streams their output. CHECK_MODULES limits the selection to
# the modules it names, or skips those named with a leading `-`, so a gate too
# large for one job splits across several.
# CHECK_FAIL_FAST=1 stops every lane at the first failed module, for the land
# gate, which ejects a failing batch either way and gains nothing from the rest.
# `--query` prints the selection before CI installs its toolchain and caches.
# `ci-platform` exists only as a query for the native platform workflows.
set -euo pipefail
Expand Down Expand Up @@ -452,6 +454,7 @@ run_lane() {
record failed
((lane_status)) || lane_status=$status
current=
[[ -z "${CHECK_FAIL_FAST:-}" ]] || return "$lane_status"
break
fi
record
Expand Down Expand Up @@ -522,10 +525,12 @@ trap 'exit 143' TERM
# run's stderr as it starts and ends, so a run cut off mid-phase, such as by a
# job timeout that never reaches this script, names the phase it stopped in.
# Every lane completes even when another fails, so one unavailable check cannot
# hide unrelated failures.
# hide unrelated failures; under CHECK_FAIL_FAST the first failure cancels the
# running lanes and starts no more.
set -m
stopping=
while true; do
while [[ -z "$lone" ]] && ((next < ${#lanes[@]} && ${#running[@]} < limit)); do
while [[ -z "$lone$stopping" ]] && ((next < ${#lanes[@]} && ${#running[@]} < limit)); do
name=${lanes[next]}
if [[ -v "alone[$name]" ]] && ((${#running[@]})); then
break
Expand All @@ -547,9 +552,13 @@ while true; do
name=${running[$pid]}
unset "running[$pid]"
[[ "$pid" != "$lone" ]] || lone=
if ((status)); then
if ((status)) && [[ -z "$stopping" ]]; then
failed+=("$name")
((failed_status)) || failed_status=$status
if [[ -n "${CHECK_FAIL_FAST:-}" ]]; then
stopping=1
cancel
fi
fi
((limit == 1)) || cat "$logs/$name.log"
done
Expand Down