From 14cb1870cc6b10428abfdc74a408a3f2674de3c5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 21:16:42 -0800 Subject: [PATCH 1/6] chore(quest): claim quest/m0/nightly/README.md Quest-Claim: b9898834c4d549d2 From ba025a5d583666d8c18b192e15e17419f59b1f2e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 21:21:35 -0800 Subject: [PATCH 2/6] fix(ci): report independent Nightly failures together Co-Authored-By: GPT-6.1 Sol --- .github/workflows/nightly.yml | 13 ++++++--- quest/m0/nightly/README.md | 47 +++++++++++++++++++++++++------- rs/justfile | 46 +++++++++++++++++++------------- sh/gh/nightly.test.py | 50 +++++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 32 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2b374b37d0d..ae8ccf23f81 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -462,30 +462,35 @@ jobs: persist-credentials: false - uses: ./.github/actions/nix - name: Rust cache + id: setup uses: ./.github/actions/rust-cache with: access-key-id: ${{ secrets.MBX_R2_ACCESS_KEY_ID }} secret-access-key: ${{ secrets.MBX_R2_SECRET_ACCESS_KEY }} + # Independent checks keep running after a failure; the job still finishes red. - name: WebSocket fallback startup benchmark run: nix develop --command just js qmux-bench - name: Browser publisher fairness benchmark + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} run: nix develop --command just js serve-bench shell: bash -leo pipefail {0} - name: Raw audio encoding benchmark + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} run: nix develop --command just js raw-audio-bench shell: bash -leo pipefail {0} - name: Audio ring benchmark + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} run: nix develop --command just js ring-bench shell: bash -leo pipefail {0} - name: Native headless playback - if: ${{ !cancelled() }} + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} run: nix develop --command just test native-play shell: bash -leo pipefail {0} env: HARNESS_RUNS: ${{ runner.temp }}/moq-test-runs - name: Audio quality - if: ${{ !cancelled() }} + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} run: nix develop --command just test audio-quality --runtimes chromium --enforce --capture shell: bash -leo pipefail {0} env: @@ -497,7 +502,7 @@ jobs: # stall voids about one AAC row in seven, which would fail most nights until # it is fixed. Kept beside the browser run so its ceilings can be re-measured. - name: Native audio quality - if: ${{ !cancelled() }} + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} run: nix develop --command just test audio-quality --runtimes native shell: bash -leo pipefail {0} env: @@ -506,7 +511,7 @@ jobs: # The run directory is the diagnosis: each process's log, the shaper's # counters, the raw samples, and every row's summary. - name: Keep the run - if: ${{ !cancelled() }} + if: ${{ !cancelled() && steps.setup.outcome == 'success' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: audio-quality-run diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index 720f0a44712..3d1578452d0 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -20,15 +20,42 @@ Decided 2026-10-27 (maintainer), from the coverage audit's triage of run [40506938673](https://github.com/moq-dev/moq/actions/runs/40506938673): more children for the causes nothing owned. -This README's own work, after the children: - -- Nightly's multi-step jobs (Features first) keep going past a failed step - and fail at the end, so one break never hides the steps after it. -- Confirm a scheduled run passes. Failures already owned elsewhere stay - there: the audio near-zero p95 row belongs to - [audio jitter target](/quest/m1/audio-jitter-target/README.md), the - macOS `receive_credit` timeout to - [test flakes](/quest/m0/test-flakes-2/README.md), and the paired bench - threshold to [bench CI](/quest/m1/bench-ci.md). +The Features recipe now runs every independent top-level check and reports all +failed commands before exiting nonzero. Audio quality's independent benchmarks +and playback checks also continue after a failure, provided setup succeeded; +cancellation still stops them. The benchmark groups already collect failures. + +Remaining work: + +- Apply the same continuation to Platform's independent checks. The scheduled + run below failed its macOS Rust step and skipped Swift, OBS, and installer + checks. Coordinate with the active + [hakari work](/quest/m0/ci/hakari.md), which owns release/platform jobs; this + attempt leaves those files untouched to avoid conflicting edits. +- Confirm a scheduled run passes after the fixes land. Use scheduled runs on + `main` only; do not dispatch Nightly. The latest inspected run, + [41483654426](https://github.com/moq-dev/moq/actions/runs/41483654426) + (2026-11-02, `dde50c699f69507ced243754c2f700a6184d1576`), failed and predates + this attempt. Features stopped at the coroutine check, hiding every check + after it. Its failure is not evidence against these continuation changes. +- Triage the remaining failures against current source and ownership before + changing them. That run also failed Firefox media, the worklet matrix + (FFmpeg source download), Rust and JS audits, Miri (missing Rust sources), + ingest fuzzing, and shipping sizes. Its gate's close-drain failure belongs to + [tokio close-drain](/quest/m0/test-flakes-2/tokio-close-drain.md). The relay + session benchmark's `Evicted` failures stay with the active + [fastest-reader work](/quest/m1/unbounded-groups/fastest-reader.md). + +Failures already owned elsewhere stay there: the audio near-zero p95 row belongs +to [audio jitter target](/quest/m1/audio-jitter-target/README.md), the macOS +`receive_credit` timeout to [test flakes](/quest/m0/test-flakes-2/README.md), and +the paired bench threshold to [bench CI](/quest/m1/bench-ci.md). Public API: none. Wire: none. + +## Waiting + +- moq-dev/moq: coordinate Platform continuation with the active hakari owner + before editing those jobs, then inspect a scheduled Nightly on `main` after + the fixes land. Completion needs a green scheduled run; run 41483654426 is + red and predates this attempt. Keep owned failures with the quests above. diff --git a/rs/justfile b/rs/justfile index 0c0cf620a67..340e7fbeccf 100644 --- a/rs/justfile +++ b/rs/justfile @@ -231,25 +231,35 @@ c-tests: # Compile the full feature matrix, including isolated combinations and their tests. features: - just rs net-features --all - just rs coroutines - just rs quic-features --all - just rs tokio-features --all - just rs media-features --all - bash sh/local.sh cargo check --locked --workspace {{ no_fuzz }} --all-targets --no-default-features - bash sh/local.sh cargo check --locked -p moq-relay --no-default-features - just rs relay-minimal - bash sh/local.sh cargo check --locked -p moq-cli --no-default-features - just rs test -p moq-cli --no-default-features --features capture,websocket -E 'test(capture_window)' - just rs test -p moq-cli --no-default-features --features capture,pipewire,websocket -E 'test(capture_window)' - bash sh/local.sh cargo check --locked -p moq-bench --no-default-features - bash sh/local.sh cargo check --locked -p moq-boy --no-default-features + #!/usr/bin/env bash + set -euo pipefail + failed=() + run() { + if ! "$@"; then failed+=("$*"); fi + } + run just rs net-features --all + run just rs coroutines + run just rs quic-features --all + run just rs tokio-features --all + run just rs media-features --all + run bash sh/local.sh cargo check --locked --workspace {{ no_fuzz }} --all-targets --no-default-features + run bash sh/local.sh cargo check --locked -p moq-relay --no-default-features + run just rs relay-minimal + run bash sh/local.sh cargo check --locked -p moq-cli --no-default-features + run just rs test -p moq-cli --no-default-features --features capture,websocket -E 'test(capture_window)' + run just rs test -p moq-cli --no-default-features --features capture,pipewire,websocket -E 'test(capture_window)' + run bash sh/local.sh cargo check --locked -p moq-bench --no-default-features + run bash sh/local.sh cargo check --locked -p moq-boy --no-default-features # Alone, since the workspace's `kio/sim` dev-dependencies would unify `kio/async` back in. - bash sh/local.sh cargo check --locked --manifest-path rs/moq-auth/Cargo.toml --lib --all-features - bash sh/local.sh cargo check --locked -p moq-hls -p moq-rtmp -p moq-rtc --no-default-features --all-targets - bash sh/local.sh cargo clippy --locked --workspace {{ no_fuzz }} --all-targets --all-features -- -D warnings - RUSTDOCFLAGS="-D warnings" bash sh/local.sh cargo doc --locked --workspace {{ no_fuzz }} --all-features --no-deps - bash sh/rs/nextest.sh cargo nextest run --locked --workspace {{ no_fuzz }} --all-targets --all-features + run bash sh/local.sh cargo check --locked --manifest-path rs/moq-auth/Cargo.toml --lib --all-features + run bash sh/local.sh cargo check --locked -p moq-hls -p moq-rtmp -p moq-rtc --no-default-features --all-targets + run bash sh/local.sh cargo clippy --locked --workspace {{ no_fuzz }} --all-targets --all-features -- -D warnings + run env RUSTDOCFLAGS="-D warnings" bash sh/local.sh cargo doc --locked --workspace {{ no_fuzz }} --all-features --no-deps + run bash sh/rs/nextest.sh cargo nextest run --locked --workspace {{ no_fuzz }} --all-targets --all-features + if ((${#failed[@]})); then + printf 'features: failed: %s\n' "${failed[@]}" >&2 + exit 1 + fi # Compile and test the explicit-time model without async helpers or sessions. [positional-arguments] diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index a3e88f90fe2..64356f2d477 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -15,6 +15,56 @@ ROOT = Path(__file__).resolve().parents[2] +class FeaturesTest(unittest.TestCase): + def run_features(self, failures=()): + runner = shutil.which("just") + (ROOT / ".tmp").mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory: + scratch = Path(directory) + calls = scratch / "calls" + for command in ("just", "cargo"): + stub = scratch / command + stub.write_text( + "#!/usr/bin/env python3\n" + "import json, os, sys\n" + "call = [os.path.basename(sys.argv[0]), *sys.argv[1:]]\n" + "with open(os.environ['FEATURE_CALLS'], 'a') as output:\n" + " output.write(json.dumps(call) + '\\n')\n" + "sys.exit(call in json.loads(os.environ['FEATURE_FAILURES']))\n" + ) + stub.chmod(0o755) + result = subprocess.run( + [runner, "rs", "features"], + cwd=ROOT, + env=dict( + os.environ, + PATH=f"{scratch}:{os.environ['PATH']}", + FEATURE_CALLS=str(calls), + FEATURE_FAILURES=json.dumps(failures), + LOCAL_SUPERVISED="1", + GITHUB_ACTIONS="false", + BASH_ENV="/dev/null", + ), + capture_output=True, + text=True, + ) + return result, [json.loads(line) for line in calls.read_text().splitlines()] + + def test_independent_checks_run_after_multiple_failures(self): + passed, expected = self.run_features() + self.assertEqual(passed.returncode, 0, passed.stderr) + # Early, middle, and final failures must all survive subsequent successes. + for failures in ([expected[0], expected[len(expected) // 2]], [expected[-1]]): + with self.subTest(failures=failures): + failed, calls = self.run_features(failures) + self.assertNotEqual(failed.returncode, 0) + self.assertEqual(calls, expected) + summary = [line for line in failed.stderr.splitlines() if line.startswith("features: failed: ")] + self.assertEqual(len(summary), len(failures)) + for failure in failures: + self.assertTrue(any(" ".join(failure) in line for line in summary)) + + class NightlyTest(unittest.TestCase): def run_group(self, group, fail="", cwd=ROOT, **environment): with tempfile.TemporaryDirectory() as directory: From 82263670a34c4fa7d5a1309d38e47692885e0963 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 21:36:31 -0800 Subject: [PATCH 3/6] fix(ci): collect nested feature check failures Co-Authored-By: GPT-6.1 Sol --- quest/m0/nightly/README.md | 5 ++- rs/justfile | 37 ++++++++++-------- sh/checks.sh | 15 ++++++++ sh/gh/nightly.sh | 12 ++---- sh/gh/nightly.test.py | 79 ++++++++++++++++++++++++++++++++++---- sh/rs/media-features.sh | 15 +++++--- sh/rs/net-features.sh | 48 ++++++++++++++--------- sh/rs/tokio-features.sh | 57 +++++++++++++++------------ 8 files changed, 184 insertions(+), 84 deletions(-) create mode 100644 sh/checks.sh diff --git a/quest/m0/nightly/README.md b/quest/m0/nightly/README.md index 3d1578452d0..90b1b404914 100644 --- a/quest/m0/nightly/README.md +++ b/quest/m0/nightly/README.md @@ -20,8 +20,9 @@ Decided 2026-10-27 (maintainer), from the coverage audit's triage of run [40506938673](https://github.com/moq-dev/moq/actions/runs/40506938673): more children for the causes nothing owned. -The Features recipe now runs every independent top-level check and reports all -failed commands before exiting nonzero. Audio quality's independent benchmarks +The Features recipe and its nested checks now run independent commands and +report every failure before exiting nonzero. The shared runner announces each +command so an error can be tied to its check. Audio quality's independent benchmarks and playback checks also continue after a failure, provided setup succeeded; cancellation still stops them. The benchmark groups already collect failures. diff --git a/rs/justfile b/rs/justfile index 340e7fbeccf..73f56c2fa15 100644 --- a/rs/justfile +++ b/rs/justfile @@ -233,10 +233,7 @@ c-tests: features: #!/usr/bin/env bash set -euo pipefail - failed=() - run() { - if ! "$@"; then failed+=("$*"); fi - } + source sh/checks.sh run just rs net-features --all run just rs coroutines run just rs quic-features --all @@ -256,10 +253,7 @@ features: run bash sh/local.sh cargo clippy --locked --workspace {{ no_fuzz }} --all-targets --all-features -- -D warnings run env RUSTDOCFLAGS="-D warnings" bash sh/local.sh cargo doc --locked --workspace {{ no_fuzz }} --all-features --no-deps run bash sh/rs/nextest.sh cargo nextest run --locked --workspace {{ no_fuzz }} --all-targets --all-features - if ((${#failed[@]})); then - printf 'features: failed: %s\n' "${failed[@]}" >&2 - exit 1 - fi + finish # Compile and test the explicit-time model without async helpers or sessions. [positional-arguments] @@ -269,9 +263,13 @@ net-features *args: # Prove the poll-only library builds compile no coroutines and document cleanly. # moq-net allows the async fns still waiting on a poll conversion. coroutines: - bash sh/rs/coroutines.sh kio -- --no-default-features --features macros,runtime,manual,serde - RUSTDOCFLAGS="-D warnings" bash sh/local.sh cargo doc --locked -p kio --no-deps --no-default-features --features macros,runtime,manual,serde - bash sh/rs/coroutines.sh moq-net --allow until --allow run_front --allow serve_front -- --no-default-features + #!/usr/bin/env bash + set -euo pipefail + source sh/checks.sh + run bash sh/rs/coroutines.sh kio -- --no-default-features --features macros,runtime,manual,serde + run env RUSTDOCFLAGS="-D warnings" bash sh/local.sh cargo doc --locked -p kio --no-deps --no-default-features --features macros,runtime,manual,serde + run bash sh/rs/coroutines.sh moq-net --allow until --allow run_front --allow serve_front -- --no-default-features + finish # Prove moq-quic leaves kio's dev-only clock runtime to its consumers, and lints without its default features; a workspace build unifies both back in. [positional-arguments] @@ -285,12 +283,17 @@ quic-features *args: echo 'usage: just rs quic-features [--all]' >&2 exit 2 fi - graph=$(cargo tree --locked -p moq-quic -e normal --prefix none) - if grep -E '^tokio ' <<<"$graph"; then - echo "moq-quic must not depend on a native runtime" >&2 - exit 1 - fi - bash sh/local.sh cargo clippy --locked -p moq-quic "${targets[@]}" --no-default-features -- -D warnings + source sh/checks.sh + dependencies() { + graph=$(cargo tree --locked -p moq-quic -e normal --prefix none) || return + if grep -E '^tokio ' <<<"$graph"; then + echo "moq-quic must not depend on a native runtime" >&2 + return 1 + fi + } + run dependencies + run bash sh/local.sh cargo clippy --locked -p moq-quic "${targets[@]}" --no-default-features -- -D warnings + finish # Compile media boundaries (--all isolates every mode) and prove expensive # native/render dependencies are absent when their features are excluded. diff --git a/sh/checks.sh b/sh/checks.sh new file mode 100644 index 00000000000..265e3605404 --- /dev/null +++ b/sh/checks.sh @@ -0,0 +1,15 @@ +# shellcheck shell=bash +# Source this in a suite, run independent commands with run, then call finish. +# Functions passed to run must return errors explicitly: Bash suppresses their +# errexit inside an if condition. External commands keep their own error handling. +failed=() +run() { + echo "checks: $*" + if ! "$@"; then failed+=("$*"); fi +} +finish() { + if ((${#failed[@]})); then + printf 'checks: failed: %s\n' "${failed[@]}" >&2 + return 1 + fi +} diff --git a/sh/gh/nightly.sh b/sh/gh/nightly.sh index eef7819bc21..c548aa59afe 100755 --- a/sh/gh/nightly.sh +++ b/sh/gh/nightly.sh @@ -51,11 +51,8 @@ if [[ -n $report ]]; then rm -f "$out/report.json" "$out/paired.json" fi -failed=() -run() { - echo "nightly: $*" - if ! "$@"; then failed+=("$*"); fi -} +# shellcheck source=sh/checks.sh +source "$(dirname "${BASH_SOURCE[0]}")/../checks.sh" criterion() { local label=$1 @@ -179,10 +176,7 @@ case "$1" in ;; esac -if ((${#failed[@]})); then - printf 'nightly: failed: %s\n' "${failed[@]}" >&2 - exit 1 -fi +finish if [[ -n $report ]]; then just gh benchmark-report "$report" "$out" "$out/report.json" diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index 64356f2d477..0ac84d442af 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -7,6 +7,7 @@ import shutil import subprocess import tempfile +import textwrap import unittest from pathlib import Path @@ -16,7 +17,7 @@ class FeaturesTest(unittest.TestCase): - def run_features(self, failures=()): + def run_features(self, failures=(), nested=False, leaks=()): runner = shutil.which("just") (ROOT / ".tmp").mkdir(exist_ok=True) with tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory: @@ -25,12 +26,46 @@ def run_features(self, failures=()): for command in ("just", "cargo"): stub = scratch / command stub.write_text( - "#!/usr/bin/env python3\n" - "import json, os, sys\n" - "call = [os.path.basename(sys.argv[0]), *sys.argv[1:]]\n" - "with open(os.environ['FEATURE_CALLS'], 'a') as output:\n" - " output.write(json.dumps(call) + '\\n')\n" - "sys.exit(call in json.loads(os.environ['FEATURE_FAILURES']))\n" + textwrap.dedent("""\ + #!/usr/bin/env python3 + import json, os, subprocess, sys + from pathlib import Path + args = sys.argv[1:] + call = [os.path.basename(sys.argv[0]), *[ + 'mir=' if arg.startswith('mir=') else arg for arg in args + ]] + with open(os.environ['FEATURE_CALLS'], 'a') as output: + output.write(json.dumps(call) + '\\n') + if call in json.loads(os.environ['FEATURE_FAILURES']): + sys.exit(1) + if call[0] == 'just': + if os.environ['FEATURE_NESTED'] == '1' and args[1] in ( + 'net-features', 'quic-features', 'tokio-features', 'media-features', 'coroutines' + ): + sys.exit(subprocess.call([os.environ['FEATURE_REAL_JUST'], *args])) + sys.exit(0) + if args[0] == 'tree': + for crate in json.loads(os.environ['FEATURE_LEAKS']): + print(crate + ' v1.0.0') + features = args[args.index('--features') + 1].split(',') if '--features' in args else [] + for feature, crates in { + 'openh264': ['openh264', 'openh264-sys2'], 'nvidia': ['moq-nvenc'], + 'render': ['wgpu'], 'vpx': ['libvpx-native-sys'], 'capture': ['cpal'], + }.items(): + if feature in features: + for crate in crates: + print(crate + ' v1.0.0') + elif args[0] == 'rustc': + package = args[args.index('-p') + 1] + for arg in args: + if arg.startswith('mir='): + Path(arg[4:]).write_text('' if package == 'kio' else '\\n'.join( + '{async fn body of ' + name + '()}' for name in ('until', 'run_front', 'serve_front') + )) + elif args[0] == 'check' and '-p' in args and args[args.index('-p') + 1] == 'moq-tokio': + print('a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature') + sys.exit(1) + """) ) stub.chmod(0o755) result = subprocess.run( @@ -41,6 +76,9 @@ def run_features(self, failures=()): PATH=f"{scratch}:{os.environ['PATH']}", FEATURE_CALLS=str(calls), FEATURE_FAILURES=json.dumps(failures), + FEATURE_REAL_JUST=runner, + FEATURE_NESTED=str(int(nested)), + FEATURE_LEAKS=json.dumps(leaks), LOCAL_SUPERVISED="1", GITHUB_ACTIONS="false", BASH_ENV="/dev/null", @@ -59,11 +97,36 @@ def test_independent_checks_run_after_multiple_failures(self): failed, calls = self.run_features(failures) self.assertNotEqual(failed.returncode, 0) self.assertEqual(calls, expected) - summary = [line for line in failed.stderr.splitlines() if line.startswith("features: failed: ")] + summary = [line for line in failed.stderr.splitlines() if line.startswith("checks: failed: ")] self.assertEqual(len(summary), len(failures)) for failure in failures: self.assertTrue(any(" ".join(failure) in line for line in summary)) + def test_nested_feature_checks_continue_after_compile_failures(self): + passed, expected = self.run_features(nested=True) + self.assertEqual(passed.returncode, 0, passed.stderr) + failures = [] + for package in ("kio", "moq-net", "moq-quic", "moq-tokio", "moq-video"): + failures.append( + next(call for call in expected if call[0] == "cargo" and call[1] != "tree" and package in call) + ) + failed, calls = self.run_features(failures, nested=True) + self.assertNotEqual(failed.returncode, 0) + self.assertEqual(calls, expected) + summary = "\n".join(line for line in failed.stderr.splitlines() if line.startswith("checks: failed: ")) + for package in ("kio", "moq-net", "moq-quic", "moq-tokio", "moq-video"): + self.assertIn(package, summary) + self.assertIn("checks: just rs coroutines", failed.stdout) + + def test_dependency_contract_failures_do_not_hide_checks(self): + passed, expected = self.run_features(nested=True) + self.assertEqual(passed.returncode, 0, passed.stderr) + failed, calls = self.run_features(nested=True, leaks=("tokio", "cpal")) + self.assertNotEqual(failed.returncode, 0) + self.assertEqual(calls, expected) + self.assertIn("must not depend on a native runtime", failed.stderr) + self.assertIn("moq-video minimal unexpectedly contains cpal", failed.stderr) + class NightlyTest(unittest.TestCase): def run_group(self, group, fail="", cwd=ROOT, **environment): diff --git a/sh/rs/media-features.sh b/sh/rs/media-features.sh index 6f27ab49c6c..7877588e999 100755 --- a/sh/rs/media-features.sh +++ b/sh/rs/media-features.sh @@ -13,12 +13,14 @@ elif [[ $# != 0 ]]; then exit 2 fi +# shellcheck source=sh/checks.sh +source "$(dirname "${BASH_SOURCE[0]}")/../checks.sh" + targets=(--tests) if ((all)); then targets=(--all-targets); fi check() { - echo "media features: cargo check $*" - cargo check --locked "${targets[@]}" "$@" + run cargo check --locked "${targets[@]}" "$@" } # Resolved for Linux, the platform every asserted dependency targets: moq-nvenc @@ -32,7 +34,7 @@ require_crate() { local graph=$1 crate=$2 label=$3 if ! grep -Eq "^${crate} v" <<<"$graph"; then echo "media features: $label must contain $crate" >&2 - exit 1 + failed+=("$label must contain $crate") fi } @@ -40,7 +42,7 @@ forbid_crate() { local graph=$1 crate=$2 label=$3 if grep -Eq "^${crate} v" <<<"$graph"; then echo "media features: $label unexpectedly contains $crate" >&2 - exit 1 + failed+=("$label unexpectedly contains $crate") fi } @@ -52,7 +54,7 @@ minimal=$(tree -p moq-video --no-default-features) for crate in openh264 openh264-sys2 wgpu cpal; do forbid_crate "$minimal" "$crate" "moq-video minimal" done -cargo nextest run --locked -p moq-video --no-default-features +run cargo nextest run --locked -p moq-video --no-default-features if ((all)); then check -p moq-video --no-default-features --features openh264; fi software=$(tree -p moq-video --no-default-features --features openh264) @@ -76,7 +78,7 @@ require_crate "$vpx" libvpx-native-sys "moq-video VP8/VP9" for crate in openh264 openh264-sys2 wgpu; do forbid_crate "$vpx" "$crate" "moq-video VP8/VP9" done -cargo nextest run --locked -p moq-video --no-default-features --features vpx -E 'test(/::vpx::/)' +run cargo nextest run --locked -p moq-video --no-default-features --features vpx -E 'test(/::vpx::/)' if ((all)); then check -p moq-video --no-default-features --features render; fi render=$(tree -p moq-video --no-default-features --features render) @@ -111,4 +113,5 @@ if ((all)); then check -p moq-audio --no-default-features --features capture,pip audio_capture=$(tree -p moq-audio --no-default-features --features capture,pipewire) require_crate "$audio_capture" cpal "moq-audio PipeWire capture" +finish echo "media features: dependency contracts ok" diff --git a/sh/rs/net-features.sh b/sh/rs/net-features.sh index a7aca80cead..835aa201964 100644 --- a/sh/rs/net-features.sh +++ b/sh/rs/net-features.sh @@ -11,23 +11,35 @@ if [[ $# != 0 && ! ($# == 1 && $1 == --all) ]]; then exit 2 fi -# Build moq-net alone: a workspace build unifies its default features back in. -# Normal dependencies exclude the simulation executor and bench-only Tokio. -minimal=$(cargo tree --locked -p moq-net --no-default-features -e normal --prefix none) -if grep -qE '^tokio ' <<<"$minimal"; then - echo "moq-net without async must not depend on a native runtime" >&2 - exit 1 -fi +# shellcheck source=sh/checks.sh +source "$(dirname "${BASH_SOURCE[0]}")/../checks.sh" -# Tests build dev-dependencies too. Reject any path that enables async before -# compiling them, so a green test run cannot hide a broken minimal build. -features=$(cargo tree --locked -p moq-net --no-default-features -e features -i moq-net) -if grep -qE 'moq-net feature "(async|default|fuzz)"' <<<"$features"; then - echo "moq-net test dependencies must not enable async" >&2 - echo "$features" >&2 - exit 1 -fi +runtime_dependencies() { + # Build moq-net alone: a workspace build unifies its default features back in. + # Normal dependencies exclude the simulation executor and bench-only Tokio. + minimal=$(cargo tree --locked -p moq-net --no-default-features -e normal --prefix none) || return + if grep -qE '^tokio ' <<<"$minimal"; then + echo "moq-net without async must not depend on a native runtime" >&2 + return 1 + fi + +} + +test_dependencies() { + # Tests build dev-dependencies too. Enabling async here would invalidate + # their proof that the minimal build works. + features=$(cargo tree --locked -p moq-net --no-default-features -e features -i moq-net) || return + if grep -qE 'moq-net feature "(async|default|fuzz)"' <<<"$features"; then + echo "moq-net test dependencies must not enable async" >&2 + echo "$features" >&2 + return 1 + fi +} + +run runtime_dependencies +run test_dependencies -cargo clippy --locked -p moq-net --no-default-features --lib -- -D warnings -RUSTDOCFLAGS="-D warnings" cargo doc --locked -p moq-net --no-default-features --no-deps -bash sh/rs/nextest.sh cargo nextest run --locked -p moq-net --no-default-features --tests +run cargo clippy --locked -p moq-net --no-default-features --lib -- -D warnings +run env RUSTDOCFLAGS="-D warnings" cargo doc --locked -p moq-net --no-default-features --no-deps +run bash sh/rs/nextest.sh cargo nextest run --locked -p moq-net --no-default-features --tests +finish diff --git a/sh/rs/tokio-features.sh b/sh/rs/tokio-features.sh index 9352b0811d1..63fb3c7c31b 100755 --- a/sh/rs/tokio-features.sh +++ b/sh/rs/tokio-features.sh @@ -18,45 +18,54 @@ elif [[ $# != 0 ]]; then fi cd "$(git rev-parse --show-toplevel)" +# shellcheck source=sh/checks.sh +source "$(dirname "${BASH_SOURCE[0]}")/../checks.sh" -cargo clippy --locked -p moq-tokio --no-default-features -- -D warnings -cargo clippy --locked -p moq-tokio --all-features -- -D warnings +run cargo clippy --locked -p moq-tokio --no-default-features -- -D warnings +run cargo clippy --locked -p moq-tokio --all-features -- -D warnings # The QUIC driver is public API for applications without a MoQ session, so its # build must not reach moq-net, the MoQ backend's dependencies, or another QUIC stack. driver=(--no-default-features --features "driver,aws-lc-rs") targets=(--tests) if ((all)); then targets=(--all-targets); fi -cargo clippy --locked -p moq-tokio "${targets[@]}" "${driver[@]}" -- -D warnings -graph=$(cargo tree --locked -p moq-tokio -e normal --prefix none "${driver[@]}") -if leaked=$(grep -E '^(moq-net|reqwest|notify|quinn[a-z-]*|noq[a-z-]*|web-transport[a-z-]*) ' <<<"$graph"); then - echo "rs: moq-tokio's driver build must not depend on:" >&2 - sort -u <<<"$leaked" >&2 - exit 1 -fi +run cargo clippy --locked -p moq-tokio "${targets[@]}" "${driver[@]}" -- -D warnings +driver_dependencies() { + graph=$(cargo tree --locked -p moq-tokio -e normal --prefix none "${driver[@]}") || return + if leaked=$(grep -E '^(moq-net|reqwest|notify|quinn[a-z-]*|noq[a-z-]*|web-transport[a-z-]*) ' <<<"$graph"); then + echo "rs: moq-tokio's driver build must not depend on:" >&2 + sort -u <<<"$leaked" >&2 + return 1 + fi +} +run driver_dependencies # Every isolated permutation, which `just rs features` asks for. if ((all)); then - cargo clippy --locked -p moq-tokio --no-default-features --features quic,aws-lc-rs -- -D warnings - cargo clippy --locked -p moq-tokio --no-default-features --features quic,ring -- -D warnings + run cargo clippy --locked -p moq-tokio --no-default-features --features quic,aws-lc-rs -- -D warnings + run cargo clippy --locked -p moq-tokio --no-default-features --features quic,ring -- -D warnings for feature in net aws-lc-rs ring tcp uds websocket webrtc; do - cargo clippy --locked -p moq-tokio --no-default-features --features "$feature" -- -D warnings + run cargo clippy --locked -p moq-tokio --no-default-features --features "$feature" -- -D warnings done fi -# shellcheck disable=SC2016 # the backticks are literal, as rustc prints them -want='a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature' -if output=$(cargo check --locked -p moq-tokio --no-default-features --features quic 2>&1); then - echo "rs: moq-tokio with only quic should reject its missing crypto provider" >&2 - exit 1 -fi -if ! grep -qF "$want" <<<"$output"; then - echo "rs: moq-tokio with only quic should fail on: $want" >&2 - echo "$output" | grep -E '^error' >&2 || true - exit 1 -fi +missing_provider() { + # shellcheck disable=SC2016 # the backticks are literal, as rustc prints them + want='a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature' + if output=$(cargo check --locked -p moq-tokio --no-default-features --features quic 2>&1); then + echo "rs: moq-tokio with only quic should reject its missing crypto provider" >&2 + return 1 + fi + if ! grep -qF "$want" <<<"$output"; then + echo "rs: moq-tokio with only quic should fail on: $want" >&2 + echo "$output" | grep -E '^error' >&2 || true + return 1 + fi +} +run missing_provider # Crypto support alone must not make a transport-less server load a QUIC identity. -bash sh/rs/nextest.sh cargo nextest run --locked -p moq-tokio --lib \ +run bash sh/rs/nextest.sh cargo nextest run --locked -p moq-tokio --lib \ --no-default-features --features tcp,aws-lc-rs \ -E 'test(no_quic_backend_needs_no_served_identity)' +finish From 5f357f8644f381971761d3cc80ea65e824808e4a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 21:43:59 -0800 Subject: [PATCH 4/6] fix(ci): preserve invalid feature dependency refusal Co-Authored-By: GPT-6.1 Sol --- sh/gh/nightly.test.py | 3 +-- sh/rs/net-features.sh | 42 +++++++++++++++++------------------------- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index 0ac84d442af..9a6928cb237 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -121,10 +121,9 @@ def test_nested_feature_checks_continue_after_compile_failures(self): def test_dependency_contract_failures_do_not_hide_checks(self): passed, expected = self.run_features(nested=True) self.assertEqual(passed.returncode, 0, passed.stderr) - failed, calls = self.run_features(nested=True, leaks=("tokio", "cpal")) + failed, calls = self.run_features(nested=True, leaks=("cpal",)) self.assertNotEqual(failed.returncode, 0) self.assertEqual(calls, expected) - self.assertIn("must not depend on a native runtime", failed.stderr) self.assertIn("moq-video minimal unexpectedly contains cpal", failed.stderr) diff --git a/sh/rs/net-features.sh b/sh/rs/net-features.sh index 835aa201964..cf0f7bc5737 100644 --- a/sh/rs/net-features.sh +++ b/sh/rs/net-features.sh @@ -11,33 +11,25 @@ if [[ $# != 0 && ! ($# == 1 && $1 == --all) ]]; then exit 2 fi -# shellcheck source=sh/checks.sh -source "$(dirname "${BASH_SOURCE[0]}")/../checks.sh" - -runtime_dependencies() { - # Build moq-net alone: a workspace build unifies its default features back in. - # Normal dependencies exclude the simulation executor and bench-only Tokio. - minimal=$(cargo tree --locked -p moq-net --no-default-features -e normal --prefix none) || return - if grep -qE '^tokio ' <<<"$minimal"; then - echo "moq-net without async must not depend on a native runtime" >&2 - return 1 - fi - -} +# Build moq-net alone: a workspace build unifies its default features back in. +# Normal dependencies exclude the simulation executor and bench-only Tokio. +minimal=$(cargo tree --locked -p moq-net --no-default-features -e normal --prefix none) +if grep -qE '^tokio ' <<<"$minimal"; then + echo "moq-net without async must not depend on a native runtime" >&2 + exit 1 +fi -test_dependencies() { - # Tests build dev-dependencies too. Enabling async here would invalidate - # their proof that the minimal build works. - features=$(cargo tree --locked -p moq-net --no-default-features -e features -i moq-net) || return - if grep -qE 'moq-net feature "(async|default|fuzz)"' <<<"$features"; then - echo "moq-net test dependencies must not enable async" >&2 - echo "$features" >&2 - return 1 - fi -} +# Tests build dev-dependencies too. Reject any path that enables async before +# compiling them, so a green test run cannot hide a broken minimal build. +features=$(cargo tree --locked -p moq-net --no-default-features -e features -i moq-net) +if grep -qE 'moq-net feature "(async|default|fuzz)"' <<<"$features"; then + echo "moq-net test dependencies must not enable async" >&2 + echo "$features" >&2 + exit 1 +fi -run runtime_dependencies -run test_dependencies +# shellcheck source=sh/checks.sh +source "$(dirname "${BASH_SOURCE[0]}")/../checks.sh" run cargo clippy --locked -p moq-net --no-default-features --lib -- -D warnings run env RUSTDOCFLAGS="-D warnings" cargo doc --locked -p moq-net --no-default-features --no-deps From 9f2585d2447d29bc4584f6e57b0d65e0325685b1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 21:44:44 -0800 Subject: [PATCH 5/6] style(ci): wrap feature fixture source Co-Authored-By: GPT-6.1 Sol --- sh/gh/nightly.test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index 9a6928cb237..6567393a3fd 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -60,10 +60,12 @@ def run_features(self, failures=(), nested=False, leaks=()): for arg in args: if arg.startswith('mir='): Path(arg[4:]).write_text('' if package == 'kio' else '\\n'.join( - '{async fn body of ' + name + '()}' for name in ('until', 'run_front', 'serve_front') + '{async fn body of ' + name + '()}' + for name in ('until', 'run_front', 'serve_front') )) elif args[0] == 'check' and '-p' in args and args[args.index('-p') + 1] == 'moq-tokio': - print('a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature') + print('a rustls QUIC backend requires a crypto provider: ' + 'enable either the `aws-lc-rs` or `ring` feature') sys.exit(1) """) ) From c40e4d0fbb3266142aea19c2dbb0980276884f62 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 5 Nov 2026 12:18:31 -0800 Subject: [PATCH 6/6] fix(ci): keep Platform's Rust tests running after a failed compile Split each Platform job's Rust step into compile and test steps, and run the macOS and Windows test recipes through the shared check runner, so a compile error or one failing test no longer hides the remaining tests. Refresh the Nightly tooling stub for main's coroutine and crypto changes. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/platform.yml | 19 +++++++++++-------- rs/justfile | 26 +++++++++++++++++--------- sh/gh/nightly.test.py | 31 +++++++++++++++++-------------- 3 files changed, 45 insertions(+), 31 deletions(-) diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml index a178c017e75..f72fcb6dd0a 100644 --- a/.github/workflows/platform.yml +++ b/.github/workflows/platform.yml @@ -106,14 +106,16 @@ jobs: - name: Setup complete id: setup run: echo "Platform tools ready" - - name: Rust compile and tests + - name: Rust compile if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.rs == 'true') }} shell: bash env: CI_BASE: ${{ github.event_name != 'pull_request' && '--all' || '' }} - run: | - just rs windows "$CI_BASE" - just rs windows-test + run: just rs windows "$CI_BASE" + - name: Rust tests + if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.rs == 'true') }} + shell: bash + run: just rs windows-test - name: C if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.c == 'true') }} shell: bash @@ -159,13 +161,14 @@ jobs: - name: Setup complete id: setup run: echo "Platform tools ready" - - name: Rust compile and tests + - name: Rust compile if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.rs == 'true') }} env: CI_BASE: ${{ github.event_name != 'pull_request' && '--all' || '' }} - run: | - just rs macos "$CI_BASE" - just rs macos-test + run: just rs macos "$CI_BASE" + - name: Rust tests + if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.rs == 'true') }} + run: just rs macos-test - name: Swift simulator check if: ${{ !cancelled() && steps.setup.outcome == 'success' && (needs.scope.outputs.swift == 'true') }} run: just swift simulator diff --git a/rs/justfile b/rs/justfile index 4446953ccc7..78828572d6e 100644 --- a/rs/justfile +++ b/rs/justfile @@ -101,9 +101,13 @@ windows $BASE="": # Test the platform-sensitive crates on Windows, independent of the diff. windows-test: - just rs test --manifest-path rs/moq-auth/Cargo.toml - just rs test -p hang -p moq-tokio -p moq-sock -p moq-relay - just rs test -p moq-video --no-default-features --features render -E 'test(/^render::export::/)' + #!/usr/bin/env bash + set -euo pipefail + source sh/checks.sh + run just rs test --manifest-path rs/moq-auth/Cargo.toml + run just rs test -p hang -p moq-tokio -p moq-sock -p moq-relay + run just rs test -p moq-video --no-default-features --features render -E 'test(/^render::export::/)' + finish # Run io_uring tests when the branch affects the relay or its dependencies. [positional-arguments] @@ -144,14 +148,18 @@ macos $BASE="": # Test the platform-sensitive crates on macOS, independent of the diff. macos-test: - just rs test --manifest-path rs/moq-auth/Cargo.toml - just rs test -p hang -p moq-tokio -p moq-sock - just rs test -p moq-sock --features fast-apple-datapath - just rs test -p moq-video --no-default-features -E 'test(encode::backend::videotoolbox::tests)' - just rs test -p moq-video --no-default-features --features render -E 'test(/^render::export::/)' + #!/usr/bin/env bash + set -euo pipefail + source sh/checks.sh + run just rs test --manifest-path rs/moq-auth/Cargo.toml + run just rs test -p hang -p moq-tokio -p moq-sock + run just rs test -p moq-sock --features fast-apple-datapath + run just rs test -p moq-video --no-default-features -E 'test(encode::backend::videotoolbox::tests)' + run just rs test -p moq-video --no-default-features --features render -E 'test(/^render::export::/)' # The runner's paravirtual Metal device proves the IOSurface import, ignored elsewhere. - just rs test -p moq-video --no-default-features --features render --lib --run-ignored only \ + run just rs test -p moq-video --no-default-features --features render --lib --run-ignored only \ -E 'test(/^render::renderer::tests::(the_metal_import_matches_the_cpu_path|imports_survive_decoder_pool_recycling)$/)' + finish # Compile and lint the capture and playback features, which the default check skips. [positional-arguments] diff --git a/sh/gh/nightly.test.py b/sh/gh/nightly.test.py index 633da762593..8ea23e81a51 100644 --- a/sh/gh/nightly.test.py +++ b/sh/gh/nightly.test.py @@ -17,7 +17,7 @@ class FeaturesTest(unittest.TestCase): - def run_features(self, failures=(), nested=False, leaks=()): + def run_features(self, failures=(), nested=False, leaks=(), recipe="features"): runner = shutil.which("just") (ROOT / ".tmp").mkdir(exist_ok=True) with tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory: @@ -40,7 +40,8 @@ def run_features(self, failures=(), nested=False, leaks=()): sys.exit(1) if call[0] == 'just': if os.environ['FEATURE_NESTED'] == '1' and args[1] in ( - 'net-features', 'quic-features', 'tokio-features', 'media-features', 'coroutines' + 'net-features', 'quic-features', 'crypto-key', 'tokio-features', 'media-features', + 'coroutines', ): sys.exit(subprocess.call([os.environ['FEATURE_REAL_JUST'], *args])) sys.exit(0) @@ -56,22 +57,14 @@ def run_features(self, failures=(), nested=False, leaks=()): for crate in crates: print(crate + ' v1.0.0') elif args[0] == 'rustc': - package = args[args.index('-p') + 1] for arg in args: if arg.startswith('mir='): - Path(arg[4:]).write_text('' if package == 'kio' else '\\n'.join( - '{async fn body of ' + name + '()}' - for name in ('until', 'run_front', 'serve_front') - )) - elif args[0] == 'check' and '-p' in args and args[args.index('-p') + 1] == 'moq-tokio': - print('a rustls QUIC backend requires a crypto provider: ' - 'enable either the `aws-lc-rs` or `ring` feature') - sys.exit(1) + Path(arg[4:]).write_text('') """) ) stub.chmod(0o755) result = subprocess.run( - [runner, "rs", "features"], + [runner, "rs", recipe], cwd=ROOT, env=dict( os.environ, @@ -104,11 +97,21 @@ def test_independent_checks_run_after_multiple_failures(self): for failure in failures: self.assertTrue(any(" ".join(failure) in line for line in summary)) + def test_platform_tests_run_after_a_failure(self): + for recipe in ("macos-test", "windows-test"): + with self.subTest(recipe=recipe): + passed, expected = self.run_features(recipe=recipe) + self.assertEqual(passed.returncode, 0, passed.stderr) + failed, calls = self.run_features([expected[0]], recipe=recipe) + self.assertNotEqual(failed.returncode, 0) + self.assertEqual(calls, expected) + self.assertIn("checks: failed: " + " ".join(expected[0]), failed.stderr) + def test_nested_feature_checks_continue_after_compile_failures(self): passed, expected = self.run_features(nested=True) self.assertEqual(passed.returncode, 0, passed.stderr) failures = [] - for package in ("kio", "moq-net", "moq-quic", "moq-tokio", "moq-video"): + for package in ("kio", "moq-net", "moq-quic", "moq-crypto", "moq-tokio", "moq-video"): failures.append( next(call for call in expected if call[0] == "cargo" and call[1] != "tree" and package in call) ) @@ -116,7 +119,7 @@ def test_nested_feature_checks_continue_after_compile_failures(self): self.assertNotEqual(failed.returncode, 0) self.assertEqual(calls, expected) summary = "\n".join(line for line in failed.stderr.splitlines() if line.startswith("checks: failed: ")) - for package in ("kio", "moq-net", "moq-quic", "moq-tokio", "moq-video"): + for package in ("kio", "moq-net", "moq-quic", "moq-crypto", "moq-tokio", "moq-video"): self.assertIn(package, summary) self.assertIn("checks: just rs coroutines", failed.stdout)