From 4b0345035353a8f7e3d8728cb6f24bb47a9f9050 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 11:43:21 -0700 Subject: [PATCH 1/2] quest: claim advisory drill mutation gate Co-Authored-By: GPT-6 From f7bf6ef2041d835904bb2d6f0a489d9a206cda90 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 15 Oct 2026 11:58:08 -0700 Subject: [PATCH 2/2] ci: prove affected drill mutations on pull requests Co-Authored-By: GPT-6 --- .github/justfile | 1 + .github/workflows/check.yml | 49 +++++++++++++++++++++++++ quest/m2/README.md | 1 - quest/m2/drill-mutation-gate.md | 63 --------------------------------- test/drill/README.md | 12 +++++++ test/drill/pr.py | 54 ++++++++++++++++++++++++++++ test/drill/pr.test.py | 60 +++++++++++++++++++++++++++++++ test/justfile | 5 +++ 8 files changed, 181 insertions(+), 64 deletions(-) delete mode 100644 quest/m2/drill-mutation-gate.md create mode 100644 test/drill/pr.py create mode 100644 test/drill/pr.test.py diff --git a/.github/justfile b/.github/justfile index 7df3cb98190..f837eb30342 100644 --- a/.github/justfile +++ b/.github/justfile @@ -16,6 +16,7 @@ test: python3 bench/relay.test.py python3 bench/profile.test.py python3 sh/gh/gst-formula.test.py + python3 test/drill/pr.test.py python3 sh/changed.test.py python3 sh/dispatch-query.test.py python3 sh/dispatch-query.test.py --bench diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index afcb74d4949..b81f255a8c5 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -155,3 +155,52 @@ jobs: CI_BASE: ${{ github.event.merge_group.base_sha }} MOQ_STRICT: 1 NEXTEST_PROFILE: ci + + drill-sensitivity: + name: Drill sensitivity + if: github.event.action != 'closed' + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + # Socket timing is still uncontrolled. Keep this advisory until the drills + # use the controlled clock, then remove this and require the stable job name. + continue-on-error: true + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + + - uses: ./.github/actions/just + + - name: Select mutations + id: scope + run: just test drill-pr select "$CI_BASE" >> "$GITHUB_OUTPUT" + env: + CI_BASE: ${{ github.event.merge_group.base_sha }} + + - name: Free disk space + if: steps.scope.outputs.needed == 'true' + uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main + with: + tool-cache: false + + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + if: steps.scope.outputs.needed == 'true' + with: + determinate: false + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + + - name: Rust cache + if: steps.scope.outputs.needed == 'true' + uses: ./.github/actions/rust-cache + + - name: Prove sensitivity + if: steps.scope.outputs.needed == 'true' + run: nix develop --command just test drill-pr run "$CI_BASE" + env: + CI_BASE: ${{ github.event.merge_group.base_sha }} + MOQ_SHAPER_SEED: "1" + NEXTEST_PROFILE: ci diff --git a/quest/m2/README.md b/quest/m2/README.md index 01356504a32..58053f7cc30 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -26,7 +26,6 @@ ranked just below deterministic TS legs, which it requires. - [Time test sweep](/quest/m2/time-test-sweep.md) - unrelated test conversion and the workspace lint ratchet - [Binding audio delay](/quest/m2/binding-surface.md) - moq-ffi and every wrapper configure and observe audio playout delay -- [Drill mutation gate](/quest/m2/drill-mutation-gate.md) - a PR touching a drill mutation's target runs that mutation's sensitivity check, advisory until the drills run on controlled time - [Window properties](/quest/m2/json-window-properties.md) - random push/pop sequences through the JS and Rust Window codecs decode to the producer's window, for readers that keep up or join late - [Pipelined requests](/quest/m2/pipeline-requests/README.md) - SUBSCRIBE and the first FETCH go out with the track-info request at every hop, so first data arrives a round trip sooner per hop - [OBS native codecs](/quest/m2/obs-moq-video/README.md) - replace FFmpeg video and audio decoding with moq-video and moq-audio, deliver GPU frames, and use native audio/video encoders diff --git a/quest/m2/drill-mutation-gate.md b/quest/m2/drill-mutation-gate.md deleted file mode 100644 index 2dd13bcaea3..00000000000 --- a/quest/m2/drill-mutation-gate.md +++ /dev/null @@ -1,63 +0,0 @@ -# [M] Drill mutations prove sensitivity on the PR that could break them - -## Goal - -A PR that stops a drill mutation from proving sensitivity turns a CI check -red before merge, rather than turning the nightly -`Tests (test drill-sensitivity)` job red after it lands. The check is -advisory until the drills run on a controlled clock, then required. - -## Plan - -Why: #5054 (merged 2026-10-08) moved the sources' release out of what -`test/drill/mutations/subscriber-leaks-broadcasts.patch` leaked. The patch -still applied, so the `--apply-only` gate in `just check` (`check:drill` in -`sh/dispatch.sh`) stayed green. The mutation no longer caused a leak, and the -nightly stayed red for 3 days until #5338. `--apply-only` catches a patch -that stops applying, not one that applies but no longer reintroduces the bug. - -Decided 2026-10-11: - -- **Mechanism:** PR CI runs `test/drill/sensitivity.sh` for just the - mutations the PR could break. Running it in local `just check` was - rejected because it costs every developer a cold snapshot build. Alerting - on a red nightly instead was rejected because it doesn't stop the merge. -- **Trigger:** a mutation runs when the PR changes a file its patch modifies - (the patch's `+++ b/` paths), the patch itself, or - `rs/moq-relay/tests/drills.rs`. A change to `test/drill/sensitivity.sh` - runs every mutation. -- **Advisory until controlled time:** the drills are not deterministic yet. - `rs/moq-relay/tests/drills.rs` uses real sockets and wall-clock sleeps and - timeouts, and the impaired lane (which `ingress-burst-never-collected` - runs) draws a random `MOQ_SHAPER_SEED` unless one is set. A mutated run - can fail for the wrong reason under load. Pin the seed in the gate anyway, - so the impaired lane's shaper decisions replay; socket timing and load - still won't. Flip the check to required once the drills - move onto [Controlled time](/quest/m1/time/README.md)'s clock. Blocking on - that was rejected so the signal exists sooner. Reversed on review: the - first pass chose required on the false premise that the drills already - ran on mocked time. -- **Cost, measured 2026-10-11:** 62 of 973 commits on `main` in the prior two - weeks touched a target file, 42 of them `rs/moq-net/src/lite/subscriber.rs`. - The 2026-10-11 nightly ran five mutations in about 18 minutes. - -Implementation guidance: - -- Shape the job so it can later become required: it always runs and passes - quickly when nothing is selected, because branch protection only accepts a - check that reports. -- Run on `merge_group` too, selecting against its base SHA like - `.github/workflows/check.yml`, so merge-queue commits get the result a - required check needs. Reuse that workflow and `sensitivity.sh`'s - mutation-name arguments rather than a parallel CI framework. -- Making the check required is a repository setting. Once the drills are on - a controlled clock, ask the maintainer to add it. -- Keep the nightly full run: it still catches drift from changes the - trigger misses, such as a refactor in a file the patch doesn't name. - -Public API: none. Wire: none. - -## Related - -- [More tests under load](/quest/m1/test-flakes-2/README.md) - the other test-reliability work in this milestone -- [Controlled time](/quest/m1/time/README.md) - puts the drills on a controlled clock, which lets this check become required diff --git a/test/drill/README.md b/test/drill/README.md index d1cf41c9af6..824abb52ef1 100644 --- a/test/drill/README.md +++ b/test/drill/README.md @@ -158,6 +158,18 @@ that stop detecting their recovery failures fail CI. `just check` also runs `--apply-only` whenever Rust or this directory changes, so a patch that no longer applies fails the PR that moved its code rather than the next nightly. +The Check workflow's advisory `Drill sensitivity` job also runs on pull requests +and merge groups. It selects mutations whose `+++ b/` target paths or patch files +changed. Changing the drill tests, sensitivity harness, or selector runs them all. +An unrelated change still reports a successful job without installing Nix or +building snapshots. PR scope uses the PR base; merge-group scope uses its base SHA. + +The job pins `MOQ_SHAPER_SEED=1`, but sockets still run on the host clock, so +load can change the outcome. Its job-level `continue-on-error` keeps the workflow +advisory. Once the drills use controlled time, remove that flag and ask the +maintainer to require `Drill sensitivity`; no repository setting changes here. +The full nightly run remains independent of PR selection. + `sensitivity.sh` removes one recovery behavior at a time and requires the drill covering it to fail. Each mutation is a patch under `mutations/`, applied to a disposable copy of the tree; the checkout it runs from is never modified. diff --git a/test/drill/pr.py b/test/drill/pr.py new file mode 100644 index 00000000000..f09a4fdf87a --- /dev/null +++ b/test/drill/pr.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Select PR-sensitive drill mutations and run the existing sensitivity harness.""" + +import argparse +from pathlib import Path +import re +import subprocess + +ROOT = Path(__file__).resolve().parents[2] + + +def select(changed: set[str], mutations: Path) -> list[str]: + patches = sorted(mutations.glob("*.patch")) + if not patches: + raise ValueError("no drill mutations found") + all_mutations = bool(changed & { + "rs/moq-relay/tests/drills.rs", + "test/drill/sensitivity.sh", + "test/drill/pr.py", + }) + selected = [] + for patch in patches: + if not re.fullmatch(r"[a-z0-9-]+", patch.stem): + raise ValueError(f"invalid mutation name: {patch.name}") + paths = set(re.findall(r"^\+\+\+ b/(.+)$", patch.read_text(), re.MULTILINE)) + if not paths: + raise ValueError(f"mutation names no target files: {patch.name}") + if all_mutations or f"test/drill/mutations/{patch.name}" in changed or changed & paths: + selected.append(patch.stem) + return selected + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=["select", "run"]) + parser.add_argument("base", nargs="?", default="") + args = parser.parse_args() + changed = subprocess.check_output( + ["bash", str(ROOT / "sh/changed.sh"), args.base], cwd=ROOT, text=True, + ).splitlines() + mutations = select(set(changed), ROOT / "test/drill/mutations") + if args.mode == "select": + # Always report, including an empty diff, so this can later be a required job. + print(f"needed={str(bool(mutations)).lower()}") + return 0 + if not mutations: + print("No affected drill mutations.") + return 0 + print("Selected drill mutations: " + ", ".join(mutations), flush=True) + return subprocess.call(["bash", str(ROOT / "test/drill/sensitivity.sh"), *mutations], cwd=ROOT) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/drill/pr.test.py b/test/drill/pr.test.py new file mode 100644 index 00000000000..945d7626d47 --- /dev/null +++ b/test/drill/pr.test.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Scope regressions for the advisory PR mutation job.""" + +import importlib.util +from pathlib import Path +import subprocess +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location("drill_pr", Path(__file__).with_name("pr.py")) +drill = importlib.util.module_from_spec(spec) +spec.loader.exec_module(drill) + + +class Scope(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.mutations = Path(self.temp.name) + (self.mutations / "first.patch").write_text("+++ b/rs/first.rs\n+++ b/rs/shared.rs\n") + (self.mutations / "second.patch").write_text("+++ b/rs/second.rs\n+++ b/rs/shared.rs\n") + + def selected(self, *changed): + return drill.select(set(changed), self.mutations) + + def test_recipe_reports_a_result_for_an_explicit_base(self): + result = subprocess.run( + ["just", "test", "drill-pr", "select", "HEAD"], cwd=drill.ROOT, + text=True, capture_output=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertRegex(result.stdout, r"^needed=(true|false)\n$") + + def test_empty_and_unrelated_changes_select_nothing(self): + self.assertEqual(self.selected(), []) + self.assertEqual(self.selected("README.md", ".github/workflows/check.yml"), []) + + def test_target_paths_and_patch_changes_select_only_affected_mutations(self): + self.assertEqual(self.selected("rs/first.rs"), ["first"]) + self.assertEqual(self.selected("test/drill/mutations/second.patch"), ["second"]) + self.assertEqual(self.selected("rs/shared.rs"), ["first", "second"]) + self.assertEqual(self.selected("rs/first.rs", "rs/shared.rs"), ["first", "second"]) + + def test_drill_or_harness_changes_select_every_mutation(self): + for path in ["rs/moq-relay/tests/drills.rs", "test/drill/sensitivity.sh", "test/drill/pr.py"]: + with self.subTest(path=path): + self.assertEqual(self.selected(path), ["first", "second"]) + + def test_malformed_mutations_fail_instead_of_silently_selecting_nothing(self): + (self.mutations / "broken.patch").write_text("not a patch\n") + with self.assertRaisesRegex(ValueError, "no target files"): + self.selected("README.md") + + def test_deleted_mutations_are_not_passed_to_the_harness(self): + (self.mutations / "second.patch").unlink() + self.assertEqual(self.selected("test/drill/mutations/second.patch"), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/justfile b/test/justfile index d3d6056e18e..f44471ecc4e 100644 --- a/test/justfile +++ b/test/justfile @@ -117,6 +117,11 @@ audio-quality-record *args: drill-sensitivity *args: ./drill/sensitivity.sh "$@" +# Select or run the PR's affected sensitivity mutations against its base. +[positional-arguments] +drill-pr mode base="": + python3 ./drill/pr.py "$@" + # Round-trips a PCR-paced TS through a relay and checks the subscriber's `export # ts` output with FFmpeg, TSDuck and a custom analyzer (the T-STD buffer model, PCR # spacing and byte schedule). Flags: `--analyze-only file.ts`