From 0d1e6ba575a953adbe102f50e31ed320db06032f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 29 Oct 2026 22:53:10 -0700 Subject: [PATCH 1/4] chore(quest): claim quest/m0/ci/five-minutes.md Quest-Claim: c013d7486f662e13 From c6c5dd45e9b69b0c434c23eeb86489f21ac40a06 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 29 Oct 2026 23:53:19 -0700 Subject: [PATCH 2/4] ci: keep each local Rust gate's feature contracts on the changed crates just check lints feature shapes and runs the crate feature contracts for the crates the diff changed; the merge queue's just ci keeps their dependents. The gate compiles the loom models in the dev profile. Co-Authored-By: Claude Opus 5.5 --- rs/justfile | 8 +++++++- sh/dispatch-query.test.py | 13 ++++++++++++ sh/dispatch.sh | 10 ++++++++- sh/rs/select.sh | 43 +++++++++++++++++++++++++++++++-------- sh/rs/select.test.py | 30 +++++++++++++++++++++++---- 5 files changed, 89 insertions(+), 15 deletions(-) diff --git a/rs/justfile b/rs/justfile index 6fe6122a0b9..d90217e483e 100644 --- a/rs/justfile +++ b/rs/justfile @@ -322,7 +322,7 @@ doctest *args: bash sh/local.sh cargo test --locked --doc "$@" # `--cfg loom` swaps kio's primitives for loom's, so it rebuilds the world and -# executes only in Nightly; the selected Rust gate compiles it with `--no-run`. +# executes only in Nightly; the selected Rust gate compiles it with `loom-check`. # `--release` because a model check runs the body once # per interleaving (411s -> 51s); the assertions are plain `assert!`. kio's # models search every interleaving; moq-net's cap preemptions (see @@ -337,6 +337,12 @@ loom *args: RUSTFLAGS="--cfg loom --cfg crossbeam_loom" cargo test --locked --release -p moq-net --test loom "$@" RUSTFLAGS="--cfg loom --cfg crossbeam_loom" cargo test --locked --release -p moq-net --test expiry-loom "$@" +# Compile the loom models without running them. The dev profile type-checks +# the same code: after a moq-net edit it took 46s against release's 146s. +loom-check: + RUSTFLAGS="--cfg loom --cfg crossbeam_loom" cargo test --locked --no-run -p kio --lib + RUSTFLAGS="--cfg loom --cfg crossbeam_loom" cargo test --locked --no-run -p moq-net --test loom --test expiry-loom + # Run the moq-quic simulation's random workloads from a fresh seed, printing the settings that replay it, then the bandwidth probe's acceptance suite. quic-sim: #!/usr/bin/env bash diff --git a/sh/dispatch-query.test.py b/sh/dispatch-query.test.py index 3fdfbe72051..15975372ea3 100644 --- a/sh/dispatch-query.test.py +++ b/sh/dispatch-query.test.py @@ -218,6 +218,19 @@ def run_ci(self, base="origin/main"): self.assertEqual(result.returncode, 0, result.stderr) return (self.root / "calls").read_text().splitlines() + def test_only_the_local_check_narrows_feature_contracts(self): + # The queue's `just ci` keeps the dependents' feature shapes. + self.change("rs/moq-net/src/lib.rs") + for action, scope in (("check", "changed"), ("ci", "")): + with self.subTest(action=action): + self.stub_ci('[[ "$1" != "rs" || "$2" != "lint-changed" ]] || ' + 'printf "scope=%s\\n" "${RS_FEATURE_SCOPE:-}" >>"$CALLS"\n') + result = subprocess.run(["bash", "sh/dispatch.sh", action, "origin/main"], + cwd=self.repo, env=self.env, text=True, capture_output=True) + self.assertEqual(result.returncode, 0, result.stderr) + calls = (self.root / "calls").read_text().splitlines() + self.assertIn(f"scope={scope}", calls) + def test_failure_keeps_running_lanes_and_starts_remaining_modules(self): # The failed lane finishes before go releases the two parked lanes. # No timing assumption can make cancellation look like completion. diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 37b1dff5443..b1d4b66ef49 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -293,7 +293,15 @@ step() { check:moq_sh) just infra moq-sh test ;; # The check keeps docs and target-specific lints; the test phase # enforces default-feature clippy without a second compile. - check:rs) just rs lint-changed "$rs_list" ;; + # Locally, the feature-contract checks skip the dependents of the + # changed crates; the queue keeps them (see sh/rs/select.sh). + check:rs) + if [[ "$action" == check ]]; then + RS_FEATURE_SCOPE=changed just rs lint-changed "$rs_list" + else + just rs lint-changed "$rs_list" + fi + ;; check:bench) python3 bench/compare.test.py bash sh/local.sh cargo check --locked --package moq-relay --package moq-bench --features moq-relay/io-uring diff --git a/sh/rs/select.sh b/sh/rs/select.sh index 1cd75fe3abf..d9d15456c12 100755 --- a/sh/rs/select.sh +++ b/sh/rs/select.sh @@ -14,7 +14,8 @@ list=${2:?$usage} cd "$(git rev-parse --show-toplevel)" -# Print `ALL`, nothing (no crate affected), or cargo package ids. +# Print `ALL`, nothing (no crate affected), or cargo package ids: the changed +# crates and their dependents, or with `seeds`, the changed crates alone. select_packages() { if [[ "$action" == uring && -f "$list" ]] && grep -qE '^(justfile|flake\.(nix|lock)|sh/changed\.sh|\.github/workflows/uring\.yml|\.cargo/.*|\.github/actions/.*)$' "$list"; then echo ALL @@ -80,7 +81,8 @@ select_packages() { # names that are not workspace crates, such as a seed from `rs/AGENTS.md`. # The seeds go through the environment because BSD awk (macOS) rejects a # newline in a `-v` value. - selected=$(SEEDS="$seeds" awk ' + selected=$seeds + [[ "${1:-}" == seeds ]] || selected=$(SEEDS="$seeds" awk ' BEGIN { split(ENVIRON["SEEDS"], s, "\n"); for (i in s) want[s[i]] = 1 } { pkg[NR] = $1; dep[NR] = $2 } END { @@ -110,13 +112,31 @@ select_packages() { packages=$(select_packages) # Ids are `path+file://#` for a directory named after its crate. -names=$(tr ' ' '\n' <<<"$packages" | sed -n 's|.*/\([^/#]*\)#.*|\1|p' | sort -u | tr '\n' ' ') +crate_names() { + tr ' ' '\n' <<<"$1" | sed -n 's|.*/\([^/#]*\)#.*|\1|p' | sort -u | tr '\n' ' ' +} +names=$(crate_names "$packages") + +# The feature-contract checks lint each crate without the features workspace +# unification adds, and a changed crate recompiles in every shape it is +# checked in. `RS_FEATURE_SCOPE=changed`, which `just check` sets, checks only +# the changed crates; the merge queue's `just ci` keeps their dependents too. +shape_packages=$packages +if [[ "${RS_FEATURE_SCOPE:-}" == changed && "$packages" != ALL && "$action" =~ ^(check|check-test|lint)$ ]]; then + shape_packages=$(select_packages seeds) +fi +shape_names=$(crate_names "$shape_packages") # True when the selection includes a crate matching the pattern. wants() { [[ "$packages" == ALL ]] || grep -qwE "$1" <<<"$names" } +# True when the feature-contract selection includes a crate matching the pattern. +shapes_want() { + [[ "$shape_packages" == ALL ]] || grep -qwE "$1" <<<"$shape_names" +} + case "$packages" in "") echo "rs: no crates affected; skipping." @@ -140,6 +160,11 @@ else # shellcheck disable=SC2206 flags=($packages) fi +shape_flags=("${flags[@]}") +if [[ "$shape_packages" != "$packages" ]]; then + # shellcheck disable=SC2206 + shape_flags=($shape_packages) +fi case "$action" in uring) @@ -149,12 +174,12 @@ case "$action" in just rs "$action" "${flags[@]}" # Workspace feature unification can hide an individual crate's missing # gates. Keep the boundary checks isolated and the full matrix nightly. - just rs feature-shapes "${flags[@]}" - if wants '(kio|moq-net)'; then just rs loom --no-run; fi - if wants moq-net; then just rs net-features; fi - if wants moq-quic; then just rs quic-features; fi - if wants moq-tokio; then just rs tokio-features; fi - if wants '(moq-video|moq-audio|moq-transcode)'; then just rs media-features; fi + if [[ -n "$shape_packages" ]]; then just rs feature-shapes "${shape_flags[@]}"; fi + if wants '(kio|moq-net)'; then just rs loom-check; fi + if shapes_want moq-net; then just rs net-features; fi + if shapes_want moq-quic; then just rs quic-features; fi + if shapes_want moq-tokio; then just rs tokio-features; fi + if shapes_want '(moq-video|moq-audio|moq-transcode)'; then just rs media-features; fi # moq-wasm's crate root is `#![cfg(target_arch = "wasm32")]`, so only a # wasm32 pass checks it. moq-mux, moq-ffi, and moq-clock ride along in that pass, and # nothing depends on moq-ffi, so each has to reach it on its own. diff --git a/sh/rs/select.test.py b/sh/rs/select.test.py index 049ef91bd13..b8034b77b61 100644 --- a/sh/rs/select.test.py +++ b/sh/rs/select.test.py @@ -15,7 +15,7 @@ class SelectTest(unittest.TestCase): # CI runs target lints before its test build. - def selected(self, *paths, action="lint", github_actions=False): + def selected(self, *paths, action="lint", github_actions=False, feature_scope=None): with tempfile.TemporaryDirectory() as directory: scratch = Path(directory) calls = scratch / "calls" @@ -27,6 +27,9 @@ def selected(self, *paths, action="lint", github_actions=False): env = dict(os.environ, LOCAL_SUPERVISED="1", PATH=f"{scratch}:{os.environ['PATH']}", SELECT_CALLS=str(calls), BASH_ENV=os.devnull, GITHUB_ACTIONS=str(github_actions).lower()) + env.pop("RS_FEATURE_SCOPE", None) + if feature_scope: + env["RS_FEATURE_SCOPE"] = feature_scope subprocess.run(["bash", "sh/rs/select.sh", action, str(changed)], cwd=ROOT, env=env, check=True, capture_output=True, text=True) return calls.read_text().splitlines() if calls.exists() else [] @@ -75,15 +78,34 @@ def test_boundary_checks_follow_reverse_dependencies(self): calls = self.selected("rs/moq-stats/benches/producer_tick.rs") self.assertFalse(any(call.endswith("-features") for call in calls), calls) + def test_local_feature_contracts_skip_dependents(self): + # `just check` lints feature shapes for the changed crates alone; their + # dependents keep the default-feature lint and the queue's shapes. + calls = self.selected("rs/kio/src/lib.rs", feature_scope="changed") + shapes = next(call for call in calls if call.startswith("rs feature-shapes ")) + self.assertIn("/rs/kio#", shapes) + self.assertNotIn("/rs/moq-net#", shapes) + self.assertTrue(any("/rs/moq-net#" in call for call in calls if call.startswith("rs lint ")), calls) + self.assertIn("rs loom-check", calls) + self.assertFalse(any(call.endswith("-features") for call in calls), calls) + for path, owner in (("rs/moq-net/src/lib.rs", "net"), ("rs/moq-tokio/src/lib.rs", "tokio"), + ("rs/moq-quic/src/lib.rs", "quic"), ("rs/moq-video/src/lib.rs", "media")): + with self.subTest(path=path): + calls = self.selected(path, feature_scope="changed") + self.assertEqual([call for call in calls if call.endswith("-features")], [f"rs {owner}-features"]) + # Inputs every crate shares still select the workspace. + self.assertIn("rs feature-shapes --workspace --exclude moq-net-fuzz --exclude moq-quic-fuzz", + self.selected("Cargo.lock", feature_scope="changed")) + def test_loom_compiles_only_when_its_model_changes(self): for action in ("check", "check-test", "lint"): for path in ("rs/kio/src/lib.rs", "rs/moq-net/tests/loom.rs", "Cargo.lock"): with self.subTest(action=action, path=path): - self.assertIn("rs loom --no-run", self.selected(path, action=action)) + self.assertIn("rs loom-check", self.selected(path, action=action)) for path in ("rs/moq-tokio/src/lib.rs", "rs/moq-stats/benches/producer_tick.rs"): with self.subTest(action=action, path=path): - self.assertNotIn("rs loom --no-run", self.selected(path, action=action)) - self.assertNotIn("rs loom --no-run", self.selected("rs/moq-net/src/lib.rs", action="test")) + self.assertNotIn("rs loom-check", self.selected(path, action=action)) + self.assertNotIn("rs loom-check", self.selected("rs/moq-net/src/lib.rs", action="test")) def test_device_features_join_the_one_test_build(self): for path, features in (("rs/moq-audio/src/lib.rs", DEVICE_FEATURES), From cb4fe98691ee927400b194ec5516f236fc1b84a5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 00:03:21 -0700 Subject: [PATCH 3/4] quest: rewrite five-minutes to what remains after the feature-contract scope Co-Authored-By: Claude Opus 5.5 --- quest/m0/ci/five-minutes.md | 42 +++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/quest/m0/ci/five-minutes.md b/quest/m0/ci/five-minutes.md index 1ad1da4429f..3a53c349b36 100644 --- a/quest/m0/ci/five-minutes.md +++ b/quest/m0/ci/five-minutes.md @@ -36,8 +36,45 @@ Every column leaves out the 177 check and 44 test runs that reached the agent's 10-minute command limit, so the percentiles and shares over 5 minutes understate the tail. -- First find which routes and steps exceed 5 minutes on representative diffs, - separating admission wait, compile, and test time. +Measured 2026-10-29 from 145 dispatch reports in agent transcripts since +10-15 (`check: seconds per module`): `check rs` is the only module whose +median exceeds 5 minutes (559 s median, 1,126 s p75, 69 of 104 runs over). +`test rs` follows (145 s median, 184 s p75); every other module's p75 is under +3 minutes. moq-net is the crate diffs touch most (39 of the last 200 commits on +`main`, 40% of those touching Rust), and its dependents span most of the +workspace, so a moq-net edit is the representative slow route. + +One moq-net edit's `rs lint-changed` on the desktop, warm mbx cache, load near +77 on 24 cores: 1,880 s. Admission waited 267 s; `feature-shapes` linted every +dependent at both feature extremes in 981 s (52%); tokio-features and +media-features, selected only as dependents, took 294 s; `cargo doc` 159 s; +the release loom compile 112 s. + +Landed in [moq-dev/moq#7282](https://github.com/moq-dev/moq/pull/7282): the +local `just check` runs `feature-shapes` and the net, QUIC, tokio, and media +feature contracts for the crates the diff changed, while the queue's `just ci` +keeps their dependents; the gate compiles the loom models in the dev profile +(46 s against release's 146 s after a moq-net edit). The same lint then took +319 s at load near 30, with `feature-shapes` at 29 s. The whole `just check` +for a moq-net edit took 547 s with load falling from 24 to 8: `check rs` +230 s and `test rs` 317 s, run one after the other in the rs lane. The test +phase is a 206 s build of moq-net's dependents and a 110 s run of 9,049 tests. + +What remains for a moq-net edit, largest first: + +- The test build recompiles every dependent after a moq-net edit. Candidates + are the epic's [thin generics](/quest/m0/ci/btree-keys.md) and + [hakari](/quest/m0/ci/hakari.md) levers, or running the rs lint beside the + test build in a second target directory, which trades CPU for wall time + against the epic's CPU measure and so needs its own decision. +- The test run is the [slow tests](/quest/m0/ci/slow-tests.md) quest's. +- The lint's remaining 230 s: net-features (62 s), the dev loom compile, + `cargo doc`, and the wasm and io_uring clippy passes. +- Measure the other routes the same way: a moq-ffi edit, which selects every + binding's lane under `CHECK_JOBS=3`, and a JS-only edit. +- `just test` has no root recipe: bare `just test` lists the cross-language + suites. Name which command the `just test` baseline above counts before + measuring it. - Prefer the epic's levers, such as [slow tests](/quest/m0/ci/slow-tests.md), [hakari](/quest/m0/ci/hakari.md), and cache hits, and tighter diff selection, over moving coverage. Move a check to the queue or nightly only under the @@ -47,4 +84,5 @@ understate the tail. ## Related - [Halve the slowest tests' CPU](/quest/m0/ci/slow-tests.md) - the largest test-time lever +- [Unify dependency features](/quest/m0/ci/hakari.md) - cache hits across feature shapes - [Land main through quest land](/quest/m0/ci/land.md) - the queue gate that keeps checks moved out of the local gate From 4ff23fa961673acd720a194186daf9a6d20b62f1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 30 Oct 2026 00:31:40 -0700 Subject: [PATCH 4/4] ci: name the landing worker in the feature-contract scope comments Co-Authored-By: Claude Opus 5.5 --- quest/m0/ci/five-minutes.md | 2 +- sh/dispatch-query.test.py | 2 +- sh/dispatch.sh | 2 +- sh/rs/select.sh | 2 +- sh/rs/select.test.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/quest/m0/ci/five-minutes.md b/quest/m0/ci/five-minutes.md index 3a53c349b36..5cb525baf9f 100644 --- a/quest/m0/ci/five-minutes.md +++ b/quest/m0/ci/five-minutes.md @@ -52,7 +52,7 @@ the release loom compile 112 s. Landed in [moq-dev/moq#7282](https://github.com/moq-dev/moq/pull/7282): the local `just check` runs `feature-shapes` and the net, QUIC, tokio, and media -feature contracts for the crates the diff changed, while the queue's `just ci` +feature contracts for the crates the diff changed, while the landing worker's `just ci` keeps their dependents; the gate compiles the loom models in the dev profile (46 s against release's 146 s after a moq-net edit). The same lint then took 319 s at load near 30, with `feature-shapes` at 29 s. The whole `just check` diff --git a/sh/dispatch-query.test.py b/sh/dispatch-query.test.py index 15975372ea3..a05fd906c73 100644 --- a/sh/dispatch-query.test.py +++ b/sh/dispatch-query.test.py @@ -219,7 +219,7 @@ def run_ci(self, base="origin/main"): return (self.root / "calls").read_text().splitlines() def test_only_the_local_check_narrows_feature_contracts(self): - # The queue's `just ci` keeps the dependents' feature shapes. + # The landing worker's `just ci` keeps the dependents' feature shapes. self.change("rs/moq-net/src/lib.rs") for action, scope in (("check", "changed"), ("ci", "")): with self.subTest(action=action): diff --git a/sh/dispatch.sh b/sh/dispatch.sh index b1d4b66ef49..42daaa39c2c 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -294,7 +294,7 @@ step() { # The check keeps docs and target-specific lints; the test phase # enforces default-feature clippy without a second compile. # Locally, the feature-contract checks skip the dependents of the - # changed crates; the queue keeps them (see sh/rs/select.sh). + # changed crates; the landing worker keeps them (see sh/rs/select.sh). check:rs) if [[ "$action" == check ]]; then RS_FEATURE_SCOPE=changed just rs lint-changed "$rs_list" diff --git a/sh/rs/select.sh b/sh/rs/select.sh index d9d15456c12..2829a8a2fde 100755 --- a/sh/rs/select.sh +++ b/sh/rs/select.sh @@ -120,7 +120,7 @@ names=$(crate_names "$packages") # The feature-contract checks lint each crate without the features workspace # unification adds, and a changed crate recompiles in every shape it is # checked in. `RS_FEATURE_SCOPE=changed`, which `just check` sets, checks only -# the changed crates; the merge queue's `just ci` keeps their dependents too. +# the changed crates; the landing worker's `just ci` keeps their dependents too. shape_packages=$packages if [[ "${RS_FEATURE_SCOPE:-}" == changed && "$packages" != ALL && "$action" =~ ^(check|check-test|lint)$ ]]; then shape_packages=$(select_packages seeds) diff --git a/sh/rs/select.test.py b/sh/rs/select.test.py index b8034b77b61..f0c1f0cb5e6 100644 --- a/sh/rs/select.test.py +++ b/sh/rs/select.test.py @@ -80,7 +80,7 @@ def test_boundary_checks_follow_reverse_dependencies(self): def test_local_feature_contracts_skip_dependents(self): # `just check` lints feature shapes for the changed crates alone; their - # dependents keep the default-feature lint and the queue's shapes. + # dependents keep the default-feature lint and the landing worker's shapes. calls = self.selected("rs/kio/src/lib.rs", feature_scope="changed") shapes = next(call for call in calls if call.startswith("rs feature-shapes ")) self.assertIn("/rs/kio#", shapes)