Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ jobs:
timeout-minutes: 60
env:
CI_BASE: --all
# Over every package, the isolated feature shapes outgrow this job beside
# the rest; release PRs run them in Features (`just gate rs_shapes`).
CHECK_MODULES: -rs_shapes

steps:
- name: Checkout
Expand Down
6 changes: 4 additions & 2 deletions .github/workflows/features.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,10 @@ jobs:
matrix:
# The queue checks selected feature extremes. Nightly and release PRs
# keep the full feature matrix and optimized artifacts; release binding
# workflows cover their optimized libraries separately.
recipe: ["rs features", "js wasm"]
# workflows cover their optimized libraries separately. `gate
# rs_shapes` is the gate's isolated extremes over every package, which
# outgrow one hosted job beside the rest of the gate.
recipe: ["rs features", "gate rs_shapes", "js wasm"]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
Expand Down
7 changes: 4 additions & 3 deletions .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -359,13 +359,14 @@ jobs:
# It also runs the bandwidth probe's acceptance suite, of which PRs run the short cases.
# `test drill impaired` runs the relay drills over a freshly seeded shaper; PRs run loopback.
# `rs cli-export` runs moq-cli's export process tests, which take tens of seconds of real time.
# `gate` runs the queue's Check over every package, catching
# what a diff-scoped queue run let into main.
# `gate -rs_shapes` runs the queue's Check over every package, catching
# what a diff-scoped queue run let into main. Features runs the rest,
# the isolated feature shapes, which outgrow one hosted job beside it.
# `rs audio-devices` plays moq-audio's device tests into a private null sink.
# `rs multicast` runs the mDNS tests, which loop multicast back on the runner.
# `rs render-sw` renders through wgpu on Mesa's software Vulkan driver.
# `test qmux` runs js/qmux against the in-tree Rust qmux, per draft.
recipe: ["gate", "rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring", "test drain", "test json-properties", "test resume-model", "rs quic-sim", "test drill impaired", "rs cli-export", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux"]
recipe: ["gate -rs_shapes", "rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring", "test drain", "test json-properties", "test resume-model", "rs quic-sim", "test drill impaired", "rs cli-export", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux"]
include:
- recipe: rs send-depth
nightly: true
Expand Down
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,8 @@ scheduling. Rust builds still use the dev shell's mbx Cargo shim.

`just check` and `just ci` run three module lanes at once, each admitted per
command; set `CHECK_JOBS` to change that, or to 1 to run them in order.
`CHECK_MODULES` runs only the modules it names, or skips each one named with a
leading `-`, as `just gate rs_shapes` and `just gate -rs_shapes` split the gate.
Local nextest runs default to at most four test workers; set
`NEXTEST_TEST_THREADS` to override it. Bun workspace scripts run sequentially
locally, preserving dependency order, while CI keeps its workspace parallelism.
Expand Down
2 changes: 1 addition & 1 deletion doc/setup/dev.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ match CI. `just check` and `just fix` refuse to run outside it; set
| `just fix` | Format and lint the packages this branch changed, and regenerate their committed JS API reports (`js/*/api/`). |
| `just check` | Compile, lint, and test the same scope. |
| `just ci` | Run the batched landing gate locally, strictly. PRs into `main` get no other CI, so run it before every push. |
| `just gate` | `just ci` over every package, as nightly and release PRs run it. |
| `just gate` | `just ci` over every package, as nightly and release PRs run it, split into the isolated feature shapes (`just gate rs_shapes`) and the rest (`just gate -rs_shapes`). |
| `just fix --all`, `just check --all` | The same, over every package. |
| `just api-diff` | Diff every published API against its last release, the JS reports against ones rendered from `origin/release`. PRs into `release` run it; `--deny` fails a backport on any break. |
| `just pub bbb <url>` | Publish Big Buck Bunny (also `tos`, `clock`, `gst`, `hls`). |
Expand Down
7 changes: 4 additions & 3 deletions justfile
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,10 @@ check $BASE="":
ci $BASE="":
CI_STRICT=1 NEXTEST_PROFILE=ci sh/dispatch.sh ci "$BASE"

# Run the queue's Linux gate over every package.
gate:
just ci --all
# Run the queue's Linux gate over every package: `just gate [MODULE|-MODULE]...`
# runs only the named modules, or skips each `-` one (see CHECK_MODULES).
gate *modules:
CHECK_MODULES={{ quote(modules) }} just ci --all

# Diff every published API against its last release: `just api-diff [--deny] [rust|js|ffi]...`.
# `--deny` fails on any break, as backports into `release` must.
Expand Down
1 change: 0 additions & 1 deletion quest/m0/nightly/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,5 @@ Public API: none. Wire: none.
- [OBS TSan passes without a wall-clock bound](/quest/m0/nightly/obs-tsan.md) - `destroySource` asserts ordering instead of `elapsed < 500ms`
- [Paired bench regressions bisected](/quest/m0/nightly/bench-regressions.md) - retirement/selection's +21-24% and two 10% rows fixed or shown to be noise
- [Firefox's player closes every VideoFrame](/quest/m0/nightly/firefox-videoframe.md) - the Media job's Firefox player stops leaking an unclosed VideoFrame
- [Gate stops hanging](/quest/m0/nightly/gate-hang.md) - `just gate` finishes instead of hanging to the timeout, and a hang names its module
- [moq-tokio's isolated feature builds pass](/quest/m0/nightly/tokio-features.md) - `Incoming::refuse` stops failing `tokio-features --all` on dead_code, so the rest of `rs features` runs
- [moq-uring's tests build on macOS and Windows](/quest/m0/nightly/kio-e0433.md) - the platform jobs stop failing with E0433 on `kio` in moq-uring's integration tests
20 changes: 0 additions & 20 deletions quest/m0/nightly/gate-hang.md

This file was deleted.

4 changes: 4 additions & 0 deletions rs/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ check-changed $LIST:
lint-changed $LIST:
sh/rs/select.sh lint "$LIST"

# `feature-shapes` for the crates in a changed-file list plus their dependents.
feature-shapes-changed $LIST:
sh/rs/select.sh feature-shapes "$LIST"

# Like `check-test`, but only the crates in a changed-file list plus their dependents.
check-test-changed $LIST:
sh/rs/select.sh check-test "$LIST"
Expand Down
42 changes: 35 additions & 7 deletions sh/dispatch-query.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,10 @@ def setUp(self):
self.env = dict(os.environ, LOCAL_SUPERVISED="1", PATH=str(self.bin), GIT_CONFIG_NOSYSTEM="1",
GIT_CONFIG_GLOBAL=os.devnull, CI_STRICT="1")
# The runner's own Actions variables would move the run's error
# annotations to stdout and append to the real job summary.
for key in ("IN_MOQ_DEV_SHELL", "ALLOW_HOST_TOOLCHAIN", "CHECK_JOBS", "GITHUB_BASE_REF",
"GITHUB_ACTIONS", "GITHUB_STEP_SUMMARY"):
# annotations to stdout and append to the real job summary, and a
# gate's CHECK_MODULES split would skip the modules these tests expect.
for key in ("IN_MOQ_DEV_SHELL", "ALLOW_HOST_TOOLCHAIN", "CHECK_JOBS", "CHECK_MODULES",
"GITHUB_BASE_REF", "GITHUB_ACTIONS", "GITHUB_STEP_SUMMARY"):
self.env.pop(key, None)
self.git("init", "-q", "-b", "main")
self.git("config", "user.name", "test")
Expand Down Expand Up @@ -188,7 +189,7 @@ def test_platform_is_only_a_query(self):
self.assertEqual(result.stdout, "")

def stub_ci(self, just=""):
for tool in ("bun", "python3", "cargo", "wasm-bindgen", "jq", "shellcheck",
for tool in ("bun", "python3", "cargo", "cargo-hack", "wasm-bindgen", "jq", "shellcheck",
"shfmt", "taplo", "nixfmt", "actionlint", "quest", "nix",
"c++", "clang-format", "cmake", "dart", "dash", "gersemi", "go",
"gradle", "java", "kramdown-rfc", "pkg-config", "uniffi-bindgen-cpp",
Expand Down Expand Up @@ -356,11 +357,17 @@ def test_terminating_the_run_stops_the_running_phase(self):
with self.subTest(jobs=jobs):
run = subprocess.Popen(["bash", "sh/dispatch.sh", "ci", "origin/main"],
cwd=self.repo, env=dict(self.env, CHECK_JOBS=jobs),
text=True, stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE)
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
started.read_text()
# A job timeout can kill the run without a signal it traps, so
# a buffered lane names its running phase before the phase ends.
os.set_blocking(run.stderr.fileno(), False)
live = (run.stderr.read() or b"").decode()
if jobs != "1":
self.assertIn("ci: check rs started", live)
run.terminate()
_, stderr = run.communicate(timeout=60)
os.set_blocking(run.stderr.fileno(), True)
stderr = live + run.communicate(timeout=60)[1].decode()
self.assertEqual(run.returncode, 143, stderr)
self.assertRegex(stderr, r"\d+ check rs \(cancelled\)")
with self.assertRaises(ProcessLookupError):
Expand All @@ -376,6 +383,27 @@ def test_ci_runs_one_lint_preamble_and_one_test_pass(self):
self.assertLess(next(i for i, call in enumerate(calls) if call.startswith("rs lint-changed ")),
next(i for i, call in enumerate(calls) if call.startswith("rs test-changed ")))

def test_feature_shapes_follow_the_rust_tests_in_their_lane(self):
self.change("rs/moq-clock/src/lib.rs")
calls = [call.split()[1] for call in self.run_ci() if call.startswith("rs ")]
self.assertEqual(calls, ["lint-changed", "test-changed", "feature-shapes-changed"])
Comment thread
kixelated marked this conversation as resolved.
Resolved

def test_module_filter_splits_the_gate(self):
modules = {"rs_shapes": ["rs feature-shapes-changed --all"],
"-rs_shapes": ["rs lint-changed --all", "rs test-changed --all"]}
for filter, expected in modules.items():
with self.subTest(filter=filter):
self.env["CHECK_MODULES"] = filter
calls = self.run_ci("--all")
self.assertEqual([call for call in calls if call.startswith("rs ")], expected)
# Every other module runs in the job that does not name one.
self.assertEqual("gh check" in calls, filter.startswith("-"))
self.env["CHECK_MODULES"] = "rs_shape"
result = subprocess.run(["bash", "sh/dispatch.sh", "ci", "--all"], cwd=self.repo,
env=self.env, text=True, capture_output=True)
self.assertEqual(result.returncode, 2, result.stderr)
self.assertIn("CHECK_MODULES names 'rs_shape'", result.stderr)

def test_flake_checks_follow_nix_inputs_instead_of_rust_source(self):
for path, expected in ((".cargo/config.toml", True),
("rs/moq-net/src/lib.rs", False),
Expand Down
51 changes: 40 additions & 11 deletions sh/dispatch.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
#!/usr/bin/env bash
# Run `just check`, `just fix`, or `just ci` over one resolved branch diff.
# check and ci run CHECK_JOBS lanes of modules at once (default 3); 1 runs
# them in order and streams their output.
# them in order and streams their output. CHECK_MODULES limits the selection to
# the modules it names, or skips those named with a leading `-`, so a gate too
# large for one job splits across several.
# `--query` prints the selection before CI installs its toolchain and caches.
# `ci-platform` exists only as a query for the native platform workflows.
set -euo pipefail
Expand Down Expand Up @@ -111,6 +113,9 @@ declare -A scope=(
[justfile]=''
[gh]=''
)
# rs's crates again, each compiled alone at its feature extremes: its own
# module because one clippy pass per package outgrows a hosted job beside rs.
scope[rs_shapes]=${scope[rs]}

# Report each native suite from the same impact map, before any toolchain is
# installed. Rust dependencies can break external consumers too, so include
Expand Down Expand Up @@ -146,6 +151,7 @@ declare -A tools=(
[workers]='bun'
[moq_sh]='dash'
[rs]='cargo jq'
[rs_shapes]='cargo cargo-hack jq'
[bench]='cargo'
[carla]='uv'
[drafts]='bun kramdown-rfc xml2rfc'
Expand Down Expand Up @@ -179,20 +185,22 @@ verb=$action
# build cache only when one is selected. JS checks build @moq/wasm for its
# consumers, and the bindings build moq-ffi.
declare -A compiles=(
[check]='rs bench js py kt go dart c cpp obs_compile'
[check]='rs rs_shapes bench js py kt go dart c cpp obs_compile'
[fix]='rs'
)

# Lanes start in this order, so the slowest come first. dart runs alone (see
# below), so it waits for the rest.
case "$action" in
check | ci) modules=(py rs c cpp obs_compile js kt flake bench carla swift go drafts markdown gh drill workers moq_sh quest obs ts_paths shell large_files toml nix justfile dart) ;;
check | ci) modules=(py rs rs_shapes c cpp obs_compile js kt flake bench carla swift go drafts markdown gh drill workers moq_sh quest obs ts_paths shell large_files toml nix justfile dart) ;;
fix) modules=(js rs py carla dart obs markdown shell toml nix justfile) ;;
esac

# Modules that share mutable state run in one lane, in module order; every
# other module is a lane of its own, and lanes run concurrently.
declare -A lane=(
# Both build the workspace in one target directory.
[rs_shapes]=rs
# `bun install` relinks the root workspace's packages even when nothing
# changed, so a concurrent install or import of them fails.
[js]=bun
Expand All @@ -218,9 +226,23 @@ declare -A lane=(
# BINDGEN_EXTRA_CLANG_ARGS, and so rebuilds what the hook then rebuilds again.
declare -A alone=([dart]=1)

# CHECK_MODULES names the modules to run, or with a leading `-` those to skip.
declare -A only=() skip=()
for word in ${CHECK_MODULES:-}; do
name=${word#-}
if [[ " ${modules[*]} " != *" $name "* ]]; then
echo "error: CHECK_MODULES names '$name', which $action does not run; pick from: ${modules[*]}" >&2
exit 2
fi
if [[ "$word" == -* ]]; then skip[$name]=1; else only[$name]=1; fi
done

selected=()
missing=()
for module in "${modules[@]}"; do
if [[ -v "skip[$module]" ]] || { ((${#only[@]})) && [[ ! -v "only[$module]" ]]; }; then
continue
fi
pattern=${scope[$module]}
if [[ -z "$all" && -n "$pattern" ]] && ! grep -qE "$pattern" "$changed"; then
continue
Expand Down Expand Up @@ -294,6 +316,7 @@ step() {
# The check keeps docs and target-specific lints; the test phase
# enforces default-feature clippy without a second compile.
check:rs) just rs lint-changed "$rs_list" ;;
check:rs_shapes) just rs feature-shapes-changed "$rs_list" ;;
check:bench)
python3 bench/compare.test.py
bash sh/local.sh cargo check --locked --package moq-relay --package moq-bench --features moq-relay/io-uring
Expand Down Expand Up @@ -372,8 +395,9 @@ github() {
}

# Run one lane's modules in order, appending each phase's seconds to the lane's
# times file. A failed phase skips the rest of that module, but independent
# modules still run. A TERM records the phase it cut short.
# times file and announcing each phase's start and end on fd 3. A failed phase
# skips the rest of that module, but independent modules still run. A TERM
# records the phase it cut short.
run_lane() {
times=$1
shift
Expand All @@ -393,6 +417,7 @@ run_lane() {
current="$phase $module"
began=$SECONDS
if github; then echo "::group::$current"; else echo "==> $current"; fi
echo "$action: $current started" >&3
# A plain statement, so errexit stays on inside the subshell.
set +e
(
Expand Down Expand Up @@ -424,7 +449,9 @@ run_lane() {
return "$lane_status"
}
record() {
echo "$((SECONDS - began)) $current${1:+ ($1)}" >>"$times"
local seconds=$((SECONDS - began))
echo "$seconds $current${1:+ ($1)}" >>"$times"
echo "$action: $current ${1:-passed} in ${seconds}s" >&3
}
cancelled() {
[[ -n "$current" ]] || return 0
Expand Down Expand Up @@ -479,8 +506,11 @@ trap 'exit 143' TERM
# Each lane runs in its own process group, so a cancel reaches its whole tree;
# a lane shell defers its own TERM trap until the running phase exits.
# Concurrent lanes buffer their output and print it whole when they finish; a
# serial run streams instead. Every lane completes even when another fails,
# so one unavailable check cannot hide unrelated failures.
# serial run streams instead. A buffered lane still announces each phase on the
# run's stderr as it starts and ends, so a run cut off mid-phase, such as by a
# job timeout that never reaches this script, names the phase it stopped in.
# Every lane completes even when another fails, so one unavailable check cannot
# hide unrelated failures.
set -m
while true; do
while [[ -z "$lone" ]] && ((next < ${#lanes[@]} && ${#running[@]} < limit)); do
Expand All @@ -491,10 +521,9 @@ while true; do
next=$((next + 1))
read -ra lane_modules <<<"${members[$name]}"
if ((limit == 1)); then
run_lane "$logs/$name.times" "${lane_modules[@]}" </dev/null &
run_lane "$logs/$name.times" "${lane_modules[@]}" </dev/null 3>/dev/null &
else
echo "$action: started ${lane_modules[*]}" >&2
run_lane "$logs/$name.times" "${lane_modules[@]}" </dev/null >"$logs/$name.log" 2>&1 &
run_lane "$logs/$name.times" "${lane_modules[@]}" </dev/null 3>&2 >"$logs/$name.log" 2>&1 &
fi
running[$!]=$name
[[ ! -v "alone[$name]" ]] || lone=$!
Expand Down
7 changes: 6 additions & 1 deletion sh/gh/queue.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ for (const job of [check.jobs.check]) {
// Release's selector and execution share one job-wide base; Land uses its
// resolved main SHA for every checked prefix, never the preceding PR's head.
assert.equal(check.jobs.check.env.CI_BASE, "--all");
// Over every package, the isolated feature shapes outgrow Check's job; Features runs them.
assert.equal(check.jobs.check.env.CHECK_MODULES, "-rs_shapes");
for (const step of check.jobs.check.steps) assert.equal(step.env?.CI_BASE, undefined, `${step.name} overrides CI_BASE`);
// biome-ignore lint/suspicious/noTemplateCurlyInString: GitHub expression is literal YAML.
assert.equal(check.concurrency.group, "check-${{ github.ref }}");
Expand Down Expand Up @@ -308,5 +310,8 @@ assert.ok(targets.some((target: string) => target.includes("-windows-")));
// Neither the feature matrix nor the optimized WASM build shares the queue's
// default-feature test artifacts, but both must block a release.
const features = await workflow("features");
assert.deepEqual(features.jobs.features.strategy.matrix.recipe, ["rs features", "js wasm"]);
assert.deepEqual(features.jobs.features.strategy.matrix.recipe, ["rs features", "gate rs_shapes", "js wasm"]);
// Nightly's gate splits across Tests and Features, which together run all of it.
assert.ok(nightly.jobs.tests.strategy.matrix.recipe.includes("gate -rs_shapes"));
assert.ok(!nightly.jobs.tests.strategy.matrix.recipe.includes("gate"));
console.log("queue event policy passed");
Loading
Loading