diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 3a7c0fe4cb8..90a6e74b0fa 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -29,6 +29,9 @@ jobs: timeout-minutes: 60 env: CI_BASE: --all + # Over every package, the isolated feature shapes outgrow this job beside + # the rest; release PRs run them in Features (`just gate rs_shapes`). + CHECK_MODULES: -rs_shapes steps: - name: Checkout diff --git a/.github/workflows/features.yml b/.github/workflows/features.yml index 5aeae576cf8..fa1309592e3 100644 --- a/.github/workflows/features.yml +++ b/.github/workflows/features.yml @@ -21,8 +21,10 @@ jobs: matrix: # The queue checks selected feature extremes. Nightly and release PRs # keep the full feature matrix and optimized artifacts; release binding - # workflows cover their optimized libraries separately. - recipe: ["rs features", "js wasm"] + # workflows cover their optimized libraries separately. `gate + # rs_shapes` is the gate's isolated extremes over every package, which + # outgrow one hosted job beside the rest of the gate. + recipe: ["rs features", "gate rs_shapes", "js wasm"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6e4936f982c..e158a9de31d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -359,13 +359,14 @@ jobs: # It also runs the bandwidth probe's acceptance suite, of which PRs run the short cases. # `test drill impaired` runs the relay drills over a freshly seeded shaper; PRs run loopback. # `rs cli-export` runs moq-cli's export process tests, which take tens of seconds of real time. - # `gate` runs the queue's Check over every package, catching - # what a diff-scoped queue run let into main. + # `gate -rs_shapes` runs the queue's Check over every package, catching + # what a diff-scoped queue run let into main. Features runs the rest, + # the isolated feature shapes, which outgrow one hosted job beside it. # `rs audio-devices` plays moq-audio's device tests into a private null sink. # `rs multicast` runs the mDNS tests, which loop multicast back on the runner. # `rs render-sw` renders through wgpu on Mesa's software Vulkan driver. # `test qmux` runs js/qmux against the in-tree Rust qmux, per draft. - recipe: ["gate", "rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring", "test drain", "test json-properties", "test resume-model", "rs quic-sim", "test drill impaired", "rs cli-export", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux"] + recipe: ["gate -rs_shapes", "rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring", "test drain", "test json-properties", "test resume-model", "rs quic-sim", "test drill impaired", "rs cli-export", "rs quic-stress", "rs audio-devices", "rs multicast", "rs render-sw", "test qmux"] include: - recipe: rs send-depth nightly: true diff --git a/README.md b/README.md index dd826af55bd..9edd37ed0be 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,8 @@ scheduling. Rust builds still use the dev shell's mbx Cargo shim. `just check` and `just ci` run three module lanes at once, each admitted per command; set `CHECK_JOBS` to change that, or to 1 to run them in order. +`CHECK_MODULES` runs only the modules it names, or skips each one named with a +leading `-`, as `just gate rs_shapes` and `just gate -rs_shapes` split the gate. Local nextest runs default to at most four test workers; set `NEXTEST_TEST_THREADS` to override it. Bun workspace scripts run sequentially locally, preserving dependency order, while CI keeps its workspace parallelism. diff --git a/doc/setup/dev.md b/doc/setup/dev.md index c48769a00f9..2f486e0e109 100644 --- a/doc/setup/dev.md +++ b/doc/setup/dev.md @@ -18,7 +18,7 @@ match CI. `just check` and `just fix` refuse to run outside it; set | `just fix` | Format and lint the packages this branch changed, and regenerate their committed JS API reports (`js/*/api/`). | | `just check` | Compile, lint, and test the same scope. | | `just ci` | Run the batched landing gate locally, strictly. PRs into `main` get no other CI, so run it before every push. | -| `just gate` | `just ci` over every package, as nightly and release PRs run it. | +| `just gate` | `just ci` over every package, as nightly and release PRs run it, split into the isolated feature shapes (`just gate rs_shapes`) and the rest (`just gate -rs_shapes`). | | `just fix --all`, `just check --all` | The same, over every package. | | `just api-diff` | Diff every published API against its last release, the JS reports against ones rendered from `origin/release`. PRs into `release` run it; `--deny` fails a backport on any break. | | `just pub bbb ` | Publish Big Buck Bunny (also `tos`, `clock`, `gst`, `hls`). | diff --git a/justfile b/justfile index 14e9e48160f..2f2bbfb84da 100644 --- a/justfile +++ b/justfile @@ -103,9 +103,10 @@ check $BASE="": ci $BASE="": CI_STRICT=1 NEXTEST_PROFILE=ci sh/dispatch.sh ci "$BASE" -# Run the queue's Linux gate over every package. -gate: - just ci --all +# Run the queue's Linux gate over every package: `just gate [MODULE|-MODULE]...` +# runs only the named modules, or skips each `-` one (see CHECK_MODULES). +gate *modules: + CHECK_MODULES={{ quote(modules) }} just ci --all # Diff every published API against its last release: `just api-diff [--deny] [rust|js|ffi]...`. # `--deny` fails on any break, as backports into `release` must. diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index 9eed52d5682..add09d3c7df 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -40,6 +40,5 @@ Public API: none. Wire: none. - [OBS TSan passes without a wall-clock bound](/quest/m0/nightly/obs-tsan.md) - `destroySource` asserts ordering instead of `elapsed < 500ms` - [Paired bench regressions bisected](/quest/m0/nightly/bench-regressions.md) - retirement/selection's +21-24% and two 10% rows fixed or shown to be noise - [Firefox's player closes every VideoFrame](/quest/m0/nightly/firefox-videoframe.md) - the Media job's Firefox player stops leaking an unclosed VideoFrame -- [Gate stops hanging](/quest/m0/nightly/gate-hang.md) - `just gate` finishes instead of hanging to the timeout, and a hang names its module - [moq-tokio's isolated feature builds pass](/quest/m0/nightly/tokio-features.md) - `Incoming::refuse` stops failing `tokio-features --all` on dead_code, so the rest of `rs features` runs - [moq-uring's tests build on macOS and Windows](/quest/m0/nightly/kio-e0433.md) - the platform jobs stop failing with E0433 on `kio` in moq-uring's integration tests diff --git a/quest/m0/nightly/gate-hang.md b/quest/m0/nightly/gate-hang.md deleted file mode 100644 index 2b1397e6036..00000000000 --- a/quest/m0/nightly/gate-hang.md +++ /dev/null @@ -1,20 +0,0 @@ -# [S] Nightly's gate stops hanging - -## Goal - -Nightly's `just gate` step finishes instead of hanging until the job timeout, -and when a test does hang, the log names the module that hung. - -## Plan - -[Run 40506938673](https://github.com/moq-dev/moq/actions/runs/40506938673/job/122198526371) -on 2026-10-27 printed nothing for its last 36 minutes before the 60-minute -timeout. Tooling-test output is buffered, so the hung module is not visible. -Found while landing [moq-dev/moq#6991](https://github.com/moq-dev/moq/pull/6991). - -Decided 2026-10-28 (maintainer): one quest for both the fix and the -visibility. Reproduce with `just gate`, fix the hung test at its cause (never -a raised timeout), and make the gate's test output identify each module as it -runs (unbuffered or per-module timestamps) so the next hang names itself. - -Public API: none. Wire: none. diff --git a/rs/justfile b/rs/justfile index 6fe6122a0b9..0bddf035c7e 100644 --- a/rs/justfile +++ b/rs/justfile @@ -69,6 +69,10 @@ check-changed $LIST: lint-changed $LIST: sh/rs/select.sh lint "$LIST" +# `feature-shapes` for the crates in a changed-file list plus their dependents. +feature-shapes-changed $LIST: + sh/rs/select.sh feature-shapes "$LIST" + # Like `check-test`, but only the crates in a changed-file list plus their dependents. check-test-changed $LIST: sh/rs/select.sh check-test "$LIST" diff --git a/sh/dispatch-query.test.py b/sh/dispatch-query.test.py index 3fdfbe72051..0e919e0eab5 100644 --- a/sh/dispatch-query.test.py +++ b/sh/dispatch-query.test.py @@ -31,9 +31,10 @@ def setUp(self): self.env = dict(os.environ, LOCAL_SUPERVISED="1", PATH=str(self.bin), GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, CI_STRICT="1") # The runner's own Actions variables would move the run's error - # annotations to stdout and append to the real job summary. - for key in ("IN_MOQ_DEV_SHELL", "ALLOW_HOST_TOOLCHAIN", "CHECK_JOBS", "GITHUB_BASE_REF", - "GITHUB_ACTIONS", "GITHUB_STEP_SUMMARY"): + # annotations to stdout and append to the real job summary, and a + # gate's CHECK_MODULES split would skip the modules these tests expect. + for key in ("IN_MOQ_DEV_SHELL", "ALLOW_HOST_TOOLCHAIN", "CHECK_JOBS", "CHECK_MODULES", + "GITHUB_BASE_REF", "GITHUB_ACTIONS", "GITHUB_STEP_SUMMARY"): self.env.pop(key, None) self.git("init", "-q", "-b", "main") self.git("config", "user.name", "test") @@ -188,7 +189,7 @@ def test_platform_is_only_a_query(self): self.assertEqual(result.stdout, "") def stub_ci(self, just=""): - for tool in ("bun", "python3", "cargo", "wasm-bindgen", "jq", "shellcheck", + for tool in ("bun", "python3", "cargo", "cargo-hack", "wasm-bindgen", "jq", "shellcheck", "shfmt", "taplo", "nixfmt", "actionlint", "quest", "nix", "c++", "clang-format", "cmake", "dart", "dash", "gersemi", "go", "gradle", "java", "kramdown-rfc", "pkg-config", "uniffi-bindgen-cpp", @@ -356,11 +357,17 @@ def test_terminating_the_run_stops_the_running_phase(self): with self.subTest(jobs=jobs): run = subprocess.Popen(["bash", "sh/dispatch.sh", "ci", "origin/main"], cwd=self.repo, env=dict(self.env, CHECK_JOBS=jobs), - text=True, stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE) + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) started.read_text() + # A job timeout can kill the run without a signal it traps, so + # a buffered lane names its running phase before the phase ends. + os.set_blocking(run.stderr.fileno(), False) + live = (run.stderr.read() or b"").decode() + if jobs != "1": + self.assertIn("ci: check rs started", live) run.terminate() - _, stderr = run.communicate(timeout=60) + os.set_blocking(run.stderr.fileno(), True) + stderr = live + run.communicate(timeout=60)[1].decode() self.assertEqual(run.returncode, 143, stderr) self.assertRegex(stderr, r"\d+ check rs \(cancelled\)") with self.assertRaises(ProcessLookupError): @@ -376,6 +383,27 @@ def test_ci_runs_one_lint_preamble_and_one_test_pass(self): self.assertLess(next(i for i, call in enumerate(calls) if call.startswith("rs lint-changed ")), next(i for i, call in enumerate(calls) if call.startswith("rs test-changed "))) + def test_feature_shapes_follow_the_rust_tests_in_their_lane(self): + self.change("rs/moq-clock/src/lib.rs") + calls = [call.split()[1] for call in self.run_ci() if call.startswith("rs ")] + self.assertEqual(calls, ["lint-changed", "test-changed", "feature-shapes-changed"]) + + def test_module_filter_splits_the_gate(self): + modules = {"rs_shapes": ["rs feature-shapes-changed --all"], + "-rs_shapes": ["rs lint-changed --all", "rs test-changed --all"]} + for filter, expected in modules.items(): + with self.subTest(filter=filter): + self.env["CHECK_MODULES"] = filter + calls = self.run_ci("--all") + self.assertEqual([call for call in calls if call.startswith("rs ")], expected) + # Every other module runs in the job that does not name one. + self.assertEqual("gh check" in calls, filter.startswith("-")) + self.env["CHECK_MODULES"] = "rs_shape" + result = subprocess.run(["bash", "sh/dispatch.sh", "ci", "--all"], cwd=self.repo, + env=self.env, text=True, capture_output=True) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn("CHECK_MODULES names 'rs_shape'", result.stderr) + def test_flake_checks_follow_nix_inputs_instead_of_rust_source(self): for path, expected in ((".cargo/config.toml", True), ("rs/moq-net/src/lib.rs", False), diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 37b1dff5443..a8ebcadfea9 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash # Run `just check`, `just fix`, or `just ci` over one resolved branch diff. # check and ci run CHECK_JOBS lanes of modules at once (default 3); 1 runs -# them in order and streams their output. +# them in order and streams their output. CHECK_MODULES limits the selection to +# the modules it names, or skips those named with a leading `-`, so a gate too +# large for one job splits across several. # `--query` prints the selection before CI installs its toolchain and caches. # `ci-platform` exists only as a query for the native platform workflows. set -euo pipefail @@ -111,6 +113,9 @@ declare -A scope=( [justfile]='' [gh]='' ) +# rs's crates again, each compiled alone at its feature extremes: its own +# module because one clippy pass per package outgrows a hosted job beside rs. +scope[rs_shapes]=${scope[rs]} # Report each native suite from the same impact map, before any toolchain is # installed. Rust dependencies can break external consumers too, so include @@ -146,6 +151,7 @@ declare -A tools=( [workers]='bun' [moq_sh]='dash' [rs]='cargo jq' + [rs_shapes]='cargo cargo-hack jq' [bench]='cargo' [carla]='uv' [drafts]='bun kramdown-rfc xml2rfc' @@ -179,20 +185,22 @@ verb=$action # build cache only when one is selected. JS checks build @moq/wasm for its # consumers, and the bindings build moq-ffi. declare -A compiles=( - [check]='rs bench js py kt go dart c cpp obs_compile' + [check]='rs rs_shapes bench js py kt go dart c cpp obs_compile' [fix]='rs' ) # Lanes start in this order, so the slowest come first. dart runs alone (see # below), so it waits for the rest. case "$action" in - check | ci) modules=(py rs c cpp obs_compile js kt flake bench carla swift go drafts markdown gh drill workers moq_sh quest obs ts_paths shell large_files toml nix justfile dart) ;; + check | ci) modules=(py rs rs_shapes c cpp obs_compile js kt flake bench carla swift go drafts markdown gh drill workers moq_sh quest obs ts_paths shell large_files toml nix justfile dart) ;; fix) modules=(js rs py carla dart obs markdown shell toml nix justfile) ;; esac # Modules that share mutable state run in one lane, in module order; every # other module is a lane of its own, and lanes run concurrently. declare -A lane=( + # Both build the workspace in one target directory. + [rs_shapes]=rs # `bun install` relinks the root workspace's packages even when nothing # changed, so a concurrent install or import of them fails. [js]=bun @@ -218,9 +226,23 @@ declare -A lane=( # BINDGEN_EXTRA_CLANG_ARGS, and so rebuilds what the hook then rebuilds again. declare -A alone=([dart]=1) +# CHECK_MODULES names the modules to run, or with a leading `-` those to skip. +declare -A only=() skip=() +for word in ${CHECK_MODULES:-}; do + name=${word#-} + if [[ " ${modules[*]} " != *" $name "* ]]; then + echo "error: CHECK_MODULES names '$name', which $action does not run; pick from: ${modules[*]}" >&2 + exit 2 + fi + if [[ "$word" == -* ]]; then skip[$name]=1; else only[$name]=1; fi +done + selected=() missing=() for module in "${modules[@]}"; do + if [[ -v "skip[$module]" ]] || { ((${#only[@]})) && [[ ! -v "only[$module]" ]]; }; then + continue + fi pattern=${scope[$module]} if [[ -z "$all" && -n "$pattern" ]] && ! grep -qE "$pattern" "$changed"; then continue @@ -294,6 +316,7 @@ step() { # The check keeps docs and target-specific lints; the test phase # enforces default-feature clippy without a second compile. check:rs) just rs lint-changed "$rs_list" ;; + check:rs_shapes) just rs feature-shapes-changed "$rs_list" ;; check:bench) python3 bench/compare.test.py bash sh/local.sh cargo check --locked --package moq-relay --package moq-bench --features moq-relay/io-uring @@ -372,8 +395,9 @@ github() { } # Run one lane's modules in order, appending each phase's seconds to the lane's -# times file. A failed phase skips the rest of that module, but independent -# modules still run. A TERM records the phase it cut short. +# times file and announcing each phase's start and end on fd 3. A failed phase +# skips the rest of that module, but independent modules still run. A TERM +# records the phase it cut short. run_lane() { times=$1 shift @@ -393,6 +417,7 @@ run_lane() { current="$phase $module" began=$SECONDS if github; then echo "::group::$current"; else echo "==> $current"; fi + echo "$action: $current started" >&3 # A plain statement, so errexit stays on inside the subshell. set +e ( @@ -424,7 +449,9 @@ run_lane() { return "$lane_status" } record() { - echo "$((SECONDS - began)) $current${1:+ ($1)}" >>"$times" + local seconds=$((SECONDS - began)) + echo "$seconds $current${1:+ ($1)}" >>"$times" + echo "$action: $current ${1:-passed} in ${seconds}s" >&3 } cancelled() { [[ -n "$current" ]] || return 0 @@ -479,8 +506,11 @@ trap 'exit 143' TERM # Each lane runs in its own process group, so a cancel reaches its whole tree; # a lane shell defers its own TERM trap until the running phase exits. # Concurrent lanes buffer their output and print it whole when they finish; a -# serial run streams instead. Every lane completes even when another fails, -# so one unavailable check cannot hide unrelated failures. +# serial run streams instead. A buffered lane still announces each phase on the +# run's stderr as it starts and ends, so a run cut off mid-phase, such as by a +# job timeout that never reaches this script, names the phase it stopped in. +# Every lane completes even when another fails, so one unavailable check cannot +# hide unrelated failures. set -m while true; do while [[ -z "$lone" ]] && ((next < ${#lanes[@]} && ${#running[@]} < limit)); do @@ -491,10 +521,9 @@ while true; do next=$((next + 1)) read -ra lane_modules <<<"${members[$name]}" if ((limit == 1)); then - run_lane "$logs/$name.times" "${lane_modules[@]}" /dev/null & else - echo "$action: started ${lane_modules[*]}" >&2 - run_lane "$logs/$name.times" "${lane_modules[@]}" "$logs/$name.log" 2>&1 & + run_lane "$logs/$name.times" "${lane_modules[@]}" &2 >"$logs/$name.log" 2>&1 & fi running[$!]=$name [[ ! -v "alone[$name]" ]] || lone=$! diff --git a/sh/gh/queue.test.ts b/sh/gh/queue.test.ts index aa6eb3365bb..cc7b76b1f43 100644 --- a/sh/gh/queue.test.ts +++ b/sh/gh/queue.test.ts @@ -34,6 +34,8 @@ for (const job of [check.jobs.check]) { // Release's selector and execution share one job-wide base; Land uses its // resolved main SHA for every checked prefix, never the preceding PR's head. assert.equal(check.jobs.check.env.CI_BASE, "--all"); +// Over every package, the isolated feature shapes outgrow Check's job; Features runs them. +assert.equal(check.jobs.check.env.CHECK_MODULES, "-rs_shapes"); for (const step of check.jobs.check.steps) assert.equal(step.env?.CI_BASE, undefined, `${step.name} overrides CI_BASE`); // biome-ignore lint/suspicious/noTemplateCurlyInString: GitHub expression is literal YAML. assert.equal(check.concurrency.group, "check-${{ github.ref }}"); @@ -308,5 +310,8 @@ assert.ok(targets.some((target: string) => target.includes("-windows-"))); // Neither the feature matrix nor the optimized WASM build shares the queue's // default-feature test artifacts, but both must block a release. const features = await workflow("features"); -assert.deepEqual(features.jobs.features.strategy.matrix.recipe, ["rs features", "js wasm"]); +assert.deepEqual(features.jobs.features.strategy.matrix.recipe, ["rs features", "gate rs_shapes", "js wasm"]); +// Nightly's gate splits across Tests and Features, which together run all of it. +assert.ok(nightly.jobs.tests.strategy.matrix.recipe.includes("gate -rs_shapes")); +assert.ok(!nightly.jobs.tests.strategy.matrix.recipe.includes("gate")); console.log("queue event policy passed"); diff --git a/sh/rs/select.sh b/sh/rs/select.sh index 1cd75fe3abf..aa5eb034329 100755 --- a/sh/rs/select.sh +++ b/sh/rs/select.sh @@ -1,14 +1,15 @@ #!/usr/bin/env bash -# Run `just rs check|check-test|lint|fix|test` on the crates a diff touches, plus -# their dependents. `platform` is the Windows and macOS compile. +# Run `just rs check|check-test|lint|feature-shapes|fix|test` on the crates a +# diff touches, plus their dependents. `platform` is the Windows and macOS +# compile. # -# Usage: sh/rs/select.sh check|check-test|lint|fix|test|platform|uring LISTFILE|--all +# Usage: sh/rs/select.sh check|check-test|lint|feature-shapes|fix|test|platform|uring LISTFILE|--all # # LISTFILE holds the changed paths, one per line. `--all` is the whole # workspace, as is a change to anything every crate is built or tested by. set -euo pipefail -usage="usage: sh/rs/select.sh check|check-test|lint|fix|test|platform|uring LISTFILE|--all" +usage="usage: sh/rs/select.sh check|check-test|lint|feature-shapes|fix|test|platform|uring LISTFILE|--all" action=${1:?$usage} list=${2:?$usage} @@ -149,7 +150,9 @@ case "$action" in just rs "$action" "${flags[@]}" # Workspace feature unification can hide an individual crate's missing # gates. Keep the boundary checks isolated and the full matrix nightly. - just rs feature-shapes "${flags[@]}" + # The dispatcher runs them as a module of their own (rs_shapes), so a + # gate split across jobs can give them one. + [[ "$action" == lint ]] || just rs feature-shapes "${flags[@]}" if wants '(kio|moq-net)'; then just rs loom --no-run; fi if wants moq-net; then just rs net-features; fi if wants moq-quic; then just rs quic-features; fi @@ -169,6 +172,9 @@ case "$action" in # The relay's io_uring listener, off the default feature set. if wants moq-relay; then just rs uring-check; fi ;; + feature-shapes) + just rs feature-shapes "${flags[@]}" + ;; fix) just rs fix "${flags[@]}" if wants '(moq-wasm|moq-mux|moq-ffi|moq-clock)'; then just rs wasm-fix; fi diff --git a/sh/rs/select.test.py b/sh/rs/select.test.py index 049ef91bd13..24be8a33802 100644 --- a/sh/rs/select.test.py +++ b/sh/rs/select.test.py @@ -55,13 +55,20 @@ def test_cargo_configuration_selects_workspace(self): def test_queue_checks_selected_feature_shapes(self): for path in ("rs/kio/src/lib.rs", "rs/moq-tokio/src/lib.rs", "rs/moq-video/src/lib.rs", "Cargo.lock", "sh/rs/nextest.sh"): - for action in ("check", "check-test", "lint"): + for action in ("check", "check-test"): with self.subTest(path=path, action=action): calls = self.selected(path, action=action) defaults = next(call for call in calls if call.startswith(f"rs {action} ")) self.assertIn(defaults.replace(f"rs {action} ", "rs feature-shapes ", 1), calls) + # The dispatcher runs lint and the shapes as separate modules. + with self.subTest(path=path, action="feature-shapes"): + lint = self.selected(path) + defaults = next(call for call in lint if call.startswith("rs lint ")) + self.assertFalse(any(call.startswith("rs feature-shapes") for call in lint), lint) + self.assertEqual(self.selected(path, action="feature-shapes"), + [defaults.replace("rs lint ", "rs feature-shapes ", 1)]) for owner in ("tokio", "media", "net"): - calls = self.selected(f"sh/rs/{owner}-features.sh") + calls = self.selected(f"sh/rs/{owner}-features.sh", action="feature-shapes") self.assertIn("rs feature-shapes --workspace --exclude moq-net-fuzz --exclude moq-quic-fuzz", calls) def test_boundary_checks_follow_reverse_dependencies(self):