From 56af393f6c3fabed41daea194fc97e73f0c4c10b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 18 Oct 2026 22:12:04 -0700 Subject: [PATCH 1/6] quest: claim quest/m0/ci/ts-path-filter Co-Authored-By: Claude Opus 5.5 From 7a1d353bb4e9a191f9679f5b8407a9263ac0c491 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 18 Oct 2026 22:15:10 -0700 Subject: [PATCH 2/6] ci(ts): derive ts.yml's crate filters from the harness's Cargo closure sh/gh/ts-paths.py resolves the workspace crates test/ts/run.sh builds with cargo tree and fails Check when ts.yml's path filters drift from that set. The filter gains the nine crates it missed (moq-archive, moq-auth, moq-clock, moq-hls, moq-rtc, moq-rtmp, moq-rtsp, moq-srt, moq-stats). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ts.yml | 30 +++++++++++++++---------- sh/dispatch.sh | 7 +++++- sh/gh/ts-paths.py | 47 ++++++++++++++++++++++++++++++++++++++++ test/ts/README.md | 3 ++- 4 files changed, 74 insertions(+), 13 deletions(-) create mode 100755 sh/gh/ts-paths.py diff --git a/.github/workflows/ts.yml b/.github/workflows/ts.yml index 2e59dcbd686..ff1e0226398 100644 --- a/.github/workflows/ts.yml +++ b/.github/workflows/ts.yml @@ -11,24 +11,32 @@ on: - "test/ts/**" - "test/lib/**" - "test/justfile" - - "rs/moq-mux/**" + # Exactly the workspace crates run.sh builds; sh/gh/ts-paths.py fails Check on drift. - "rs/hang/**" - - "rs/moq-cli/**" - - "rs/moq-net/**" - - "rs/moq-tokio/**" - - "rs/moq-relay/**" - - "rs/moq-loc/**" - - "rs/moq-json/**" - # The lossy-path step's transport and shaper. - - "rs/moq-quic/**" - - "rs/moq-shaper/**" - # Workspace dependencies of moq-net, moq-tokio and moq-mux on the TS path. - "rs/kio/**" + - "rs/moq-archive/**" + - "rs/moq-auth/**" + - "rs/moq-cli/**" + - "rs/moq-clock/**" - "rs/moq-flate/**" + - "rs/moq-hls/**" - "rs/moq-ice/**" + - "rs/moq-json/**" + - "rs/moq-loc/**" - "rs/moq-msf/**" + - "rs/moq-mux/**" + - "rs/moq-net/**" - "rs/moq-pattern/**" + - "rs/moq-quic/**" + - "rs/moq-relay/**" + - "rs/moq-rtc/**" + - "rs/moq-rtmp/**" + - "rs/moq-rtsp/**" + - "rs/moq-shaper/**" - "rs/moq-sock/**" + - "rs/moq-srt/**" + - "rs/moq-stats/**" + - "rs/moq-tokio/**" - "Cargo.toml" - "Cargo.lock" - "rust-toolchain.toml" diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 87061567699..ba682b78fc7 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -94,6 +94,9 @@ declare -A scope=( # nixpkgs, and the last moves on a flake.lock bump alone. [obs]='^(cpp/obs/|sh/obs/|flake\.(nix|lock)$)' [flake]='(^rs/|^Cargo\.(toml|lock)$|^flake\.lock$|\.nix$)' + # ts.yml's crate filters against the closure test/ts/run.sh builds, which + # moves only with a manifest, the lockfile, or the harness's packages. + [ts_paths]='^(rs/.*Cargo\.toml$|Cargo\.(toml|lock)$|test/ts/run\.sh$|\.github/workflows/ts\.yml$|sh/gh/ts-paths\.py$)' [markdown]='' [shell]='' [toml]='' @@ -149,6 +152,7 @@ declare -A tools=( [obs_compile]='cargo cmake pkg-config uniffi-bindgen-cpp' [obs]='clang-format cmake gersemi jq' [flake]='nix' + [ts_paths]='cargo python3' [markdown]='bun' [shell]='shellcheck shfmt' [toml]='taplo' @@ -171,7 +175,7 @@ declare -A compiles=( ) case "$action" in - check | ci-check) modules=(js workers moq_sh drafts rs bench quest drill py kt swift go dart cpp obs_compile obs flake markdown shell toml nix justfile gh) ;; + check | ci-check) modules=(js workers moq_sh drafts rs bench quest drill py kt swift go dart cpp obs_compile obs flake ts_paths markdown shell toml nix justfile gh) ;; fix) modules=(js rs py dart obs markdown shell toml nix justfile) ;; ci-test) modules=(js rs py) ;; esac @@ -251,6 +255,7 @@ for module in "${selected[@]}"; do check:drill) just test drill-sensitivity --apply-only ;; check:obs_compile) just obs compile ;; check:flake) nix flake check ;; + check:ts_paths) python3 sh/gh/ts-paths.py ;; check:markdown) bun install --frozen-lockfile sh/markdown.sh check diff --git a/sh/gh/ts-paths.py b/sh/gh/ts-paths.py new file mode 100755 index 00000000000..dcac58c495f --- /dev/null +++ b/sh/gh/ts-paths.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Fail when ts.yml's crate path filters differ from the workspace crates test/ts/run.sh builds. + +The filter covers exactly the activated closure of run.sh's `-p` packages with their +default features: a crate outside it must not trigger the expensive workflow, and a +crate inside it must. crates.io dependencies are covered by the Cargo.lock filter. +""" + +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github/workflows/ts.yml" +HARNESS = ROOT / "test/ts/run.sh" +# ts.yml's ubuntu-latest runner, so target-specific dependencies resolve the same on any host. +TARGET = "x86_64-unknown-linux-gnu" + +packages = sorted(set(re.findall(r"(? Date: Sun, 18 Oct 2026 22:15:27 -0700 Subject: [PATCH 3/6] quest: complete quest/m0/ci/ts-path-filter Co-Authored-By: Claude Opus 5.5 --- quest/m0/ci/README.md | 1 - quest/m0/ci/ts-path-filter.md | 25 ------------------- quest/m1/ts-reliability/grader-determinism.md | 2 +- 3 files changed, 1 insertion(+), 27 deletions(-) delete mode 100644 quest/m0/ci/ts-path-filter.md diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index 92b967a3dd3..54451fb5dda 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -67,5 +67,4 @@ The historical samples locate candidates; they are not controlled benchmarks. - [Require platform gates](/quest/m0/ci/platform-queue.md) - activate the consolidated platform checks in the main and release rulesets after the workflow lands - [mbx on R2](/quest/m0/ci/mbx-r2.md) - Rust builds on every OS read an R2 cache that only `main` writes, escaping the 10 GB Actions cache - [Nix cache](/quest/m0/ci/nix-cache.md) - `nix.moq.dev` on R2 replaces Cachix for CI and releases -- [TS path filter](/quest/m0/ci/ts-path-filter.md) - TS validation runs whenever a crate it builds changes, checked against `cargo metadata` - [First shared FFI release](/quest/m0/ci/nightly-ffi-reuse.md) - condition: the first `moq-ffi-v*` tag that runs `Release FFI`, and a `Shipping sizes` run; verify the wrappers and size keys diff --git a/quest/m0/ci/ts-path-filter.md b/quest/m0/ci/ts-path-filter.md deleted file mode 100644 index 7535bfd6441..00000000000 --- a/quest/m0/ci/ts-path-filter.md +++ /dev/null @@ -1,25 +0,0 @@ -# [XS] TS validation runs whenever a crate it builds changes - -## Goal - -`.github/workflows/ts.yml` triggers on every workspace crate the TS harness -builds, and a lint fails Check when its path filter drifts from that set. - -## Plan - -`test/ts/run.sh` builds `moq-relay`, `moq-cli` and `moq-shaper` with their -default features. That build compiles workspace crates the filter misses, -found 2026-10-17 with `cargo tree` over those roots: `moq-clock`, -`moq-auth`, `moq-stats`, `moq-archive`, `moq-hls`, `moq-rtc`, `moq-rtmp` and -`moq-srt`. Optional crates such as `moq-transcode`, `moq-video` and -`moq-nvenc` are not compiled and must not trigger the workflow. -crates.io dependencies such as `web-transport-*` are already covered by the -`Cargo.lock` filter. - -Decided 2026-10-17 (maintainer): derive the set from Cargo in a lint that -runs in Check, rather than hand-listing paths that drift with the next -dependency. Resolve it for the harness's exact packages and features (the -activated closure), not the whole workspace graph, or unrelated changes will -trigger this expensive workflow. - -Public API: none. Wire: none. diff --git a/quest/m1/ts-reliability/grader-determinism.md b/quest/m1/ts-reliability/grader-determinism.md index 2a1b19dccbd..d72deeb1cb5 100644 --- a/quest/m1/ts-reliability/grader-determinism.md +++ b/quest/m1/ts-reliability/grader-determinism.md @@ -21,7 +21,7 @@ Two sources of run-to-run variance, found by #5810 and #5854: repeated timestamp read as a hole; `decode.py` has the same exposure. Decided 2026-10-17 (maintainer): one quest for the grader, with the CI path -filter as a separate [m0 quest](/quest/m0/ci/ts-path-filter.md). The harness +filter as a separate m0 quest. The harness never gates on a random-loss percentage (see [bufferbloat](/quest/m1/bufferbloat.md)). From 46526e449baa296d5652bfff0c314b94d55b4ebf Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 18 Oct 2026 22:30:42 -0700 Subject: [PATCH 4/6] fix(ci): accept Windows package paths in the ts.yml filter lint Cargo prints `C:\...` package paths on Windows, including under MSYS and Cygwin, so the POSIX-only regex matched nothing and every filter was reported as extra. Strip cargo's own workspace_root instead of resolving against the script's path, and cover both path styles in a nightly test. Co-Authored-By: Claude Opus 5.5 --- .github/justfile | 1 + sh/gh/ts-paths.py | 73 ++++++++++++++++++++++++++---------------- sh/gh/ts-paths.test.py | 30 +++++++++++++++++ 3 files changed, 76 insertions(+), 28 deletions(-) create mode 100644 sh/gh/ts-paths.test.py diff --git a/.github/justfile b/.github/justfile index 696bd87a948..38aaa244d6b 100644 --- a/.github/justfile +++ b/.github/justfile @@ -34,6 +34,7 @@ test: python3 sh/rs/model-markers.test.py python3 sh/gh/nightly.test.py python3 sh/gh/bench-report.test.py + python3 sh/gh/ts-paths.test.py sh/go/publish-wrapper.test.sh bash .claude/hooks/direnv.test.sh python3 sh/rs/uring.test.py diff --git a/sh/gh/ts-paths.py b/sh/gh/ts-paths.py index dcac58c495f..24536190b8d 100755 --- a/sh/gh/ts-paths.py +++ b/sh/gh/ts-paths.py @@ -6,6 +6,7 @@ crate inside it must. crates.io dependencies are covered by the Cargo.lock filter. """ +import json import re import subprocess import sys @@ -17,31 +18,47 @@ # ts.yml's ubuntu-latest runner, so target-specific dependencies resolve the same on any host. TARGET = "x86_64-unknown-linux-gnu" -packages = sorted(set(re.findall(r"(? set[str]: + """Workspace-relative directories of the local packages in `cargo tree --format {p}` output. + + Both arguments come from cargo, so they share its path syntax, which is `C:\\...` on Windows + even under MSYS or Cygwin. + """ + root = workspace_root.replace("\\", "/").rstrip("/") + "/" + sources = (source.replace("\\", "/") for source in re.findall(r"^\S+ v\S+ \(([^)]+)\)", tree, re.MULTILINE)) + return {source.removeprefix(root) for source in sources if source.startswith(root)} + + +def cargo(*args: str) -> str: + return subprocess.run(["cargo", *args], cwd=ROOT, check=True, stdout=subprocess.PIPE, text=True).stdout + + +def main() -> None: + packages = sorted(set(re.findall(r"(? Date: Sun, 18 Oct 2026 22:41:34 -0700 Subject: [PATCH 5/6] fix(ci): report ts.yml filters for removed crates as extra The lint kept only filters whose Cargo.toml exists, so a filter left behind by a deleted crate was ignored instead of reported. Treat every `rs//**` filter as a crate filter. Co-Authored-By: Claude Opus 5.5 --- sh/gh/ts-paths.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/sh/gh/ts-paths.py b/sh/gh/ts-paths.py index 24536190b8d..5e2787fa215 100755 --- a/sh/gh/ts-paths.py +++ b/sh/gh/ts-paths.py @@ -44,12 +44,8 @@ def main() -> None: workspace_root = json.loads(cargo("metadata", "--locked", "--no-deps", "--format-version", "1"))["workspace_root"] built = built_dirs(tree, workspace_root) - # Every `dir/**` filter on a crate directory; other filters are the harness's own inputs. - filtered = { - path - for path in re.findall(r'^\s+- "([^"*]+)/\*\*"$', WORKFLOW.read_text(), re.MULTILINE) - if (ROOT / path / "Cargo.toml").is_file() - } + # Every `rs//**` filter, even one whose crate is gone; other filters are the harness's own inputs. + filtered = set(re.findall(r'^\s+- "(rs/[^"*]+)/\*\*"$', WORKFLOW.read_text(), re.MULTILINE)) missing, extra = sorted(built - filtered), sorted(filtered - built) for path in missing: From 796eb76c668b6b913a467b8e2bd1b0e844049178 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 18 Oct 2026 23:51:15 -0700 Subject: [PATCH 6/6] fix(ci): match ts.yml lint crates by name, not printed path Parsing the source path out of `cargo tree` text broke on checkout paths containing `)`. Match each tree line's name and version against the workspace members from `cargo metadata`, which supplies their manifest paths as JSON, and cover a parenthesized checkout path in the test. Co-Authored-By: Claude Opus 5.5 --- sh/gh/ts-paths.py | 22 +++++++++++++--------- sh/gh/ts-paths.test.py | 40 ++++++++++++++++++++++++++++------------ 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/sh/gh/ts-paths.py b/sh/gh/ts-paths.py index 5e2787fa215..7a918e4792a 100755 --- a/sh/gh/ts-paths.py +++ b/sh/gh/ts-paths.py @@ -19,15 +19,20 @@ TARGET = "x86_64-unknown-linux-gnu" -def built_dirs(tree: str, workspace_root: str) -> set[str]: - """Workspace-relative directories of the local packages in `cargo tree --format {p}` output. +def built_dirs(tree: str, metadata: dict) -> set[str]: + """Workspace-relative directories of the workspace packages in `cargo tree --format {p}` output. - Both arguments come from cargo, so they share its path syntax, which is `C:\\...` on Windows - even under MSYS or Cygwin. + Packages are matched by name and version, never by the printed source path, which can contain + spaces or parentheses. Paths come from `cargo metadata --no-deps`, in cargo's own syntax, which + is `C:\\...` on Windows even under MSYS or Cygwin. """ - root = workspace_root.replace("\\", "/").rstrip("/") + "/" - sources = (source.replace("\\", "/") for source in re.findall(r"^\S+ v\S+ \(([^)]+)\)", tree, re.MULTILINE)) - return {source.removeprefix(root) for source in sources if source.startswith(root)} + root = metadata["workspace_root"].replace("\\", "/").rstrip("/") + "/" + members = { + (package["name"], f"v{package['version']}"): + package["manifest_path"].replace("\\", "/").removeprefix(root).removesuffix("/Cargo.toml") + for package in metadata["packages"] + } + return {members[key] for key in (tuple(line.split()[:2]) for line in tree.splitlines()) if key in members} def cargo(*args: str) -> str: @@ -41,8 +46,7 @@ def main() -> None: tree = cargo("tree", "--locked", "--target", TARGET, "--edges", "normal,build", "--prefix", "none", "--format", "{p}", *(arg for p in packages for arg in ("-p", p))) - workspace_root = json.loads(cargo("metadata", "--locked", "--no-deps", "--format-version", "1"))["workspace_root"] - built = built_dirs(tree, workspace_root) + built = built_dirs(tree, json.loads(cargo("metadata", "--locked", "--no-deps", "--format-version", "1"))) # Every `rs//**` filter, even one whose crate is gone; other filters are the harness's own inputs. filtered = set(re.findall(r'^\s+- "(rs/[^"*]+)/\*\*"$', WORKFLOW.read_text(), re.MULTILINE)) diff --git a/sh/gh/ts-paths.test.py b/sh/gh/ts-paths.test.py index f41d20c3096..7ac0de87eec 100644 --- a/sh/gh/ts-paths.test.py +++ b/sh/gh/ts-paths.test.py @@ -1,4 +1,4 @@ -"""Parse cargo tree output into workspace crate directories on POSIX and Windows hosts.""" +"""Map cargo tree output to workspace crate directories whatever the checkout path looks like.""" import importlib.util from pathlib import Path @@ -9,21 +9,37 @@ spec.loader.exec_module(module) +def metadata(root: str, sep: str) -> dict: + return { + "workspace_root": root, + "packages": [ + {"name": name, "version": version, "manifest_path": sep.join([root, "rs", name, "Cargo.toml"])} + for name, version in (("moq-cli", "0.1.0"), ("moq-lite", "0.2.0"), ("moq-macro", "0.1.0"), ("moq-unused", "0.1.0")) + ], + } + + class BuiltDirsTest(unittest.TestCase): + def check(self, root: str, sep: str): + tree = "".join(f"{line}\n" for line in ( + f"moq-cli v0.1.0 ({sep.join([root, 'rs', 'moq-cli'])})", + f"moq-lite v0.2.0 ({sep.join([root, 'rs', 'moq-lite'])}) (*)", + f"moq-macro v0.1.0 ({sep.join([root, 'rs', 'moq-macro'])}) (proc-macro)", + "moq-lite v0.1.0", + "serde v1.0.0", + "serde_derive v1.0.0 (proc-macro)", + "forked v0.1.0 (https://github.com/example/forked#abc123)", + )) + self.assertEqual(module.built_dirs(tree, metadata(root, sep)), {"rs/moq-cli", "rs/moq-lite", "rs/moq-macro"}) + def test_posix(self): - tree = ( - "moq-cli v0.1.0 (/repo/rs/moq-cli)\n" - "moq-lite v0.2.0 (/repo/rs/moq-lite) (*)\n" - "moq-macro v0.1.0 (/repo/rs/moq-macro) (proc-macro)\n" - "serde v1.0.0\n" - "serde_derive v1.0.0 (proc-macro)\n" - "forked v0.1.0 (https://github.com/example/forked#abc123)\n" - ) - self.assertEqual(module.built_dirs(tree, "/repo"), {"rs/moq-cli", "rs/moq-lite", "rs/moq-macro"}) + self.check("/repo", "/") def test_windows(self): - tree = "moq-cli v0.1.0 (C:\\repo\\rs\\moq-cli)\nmoq-lite v0.2.0 (C:\\repo\\rs\\moq-lite) (*)\nserde v1.0.0\n" - self.assertEqual(module.built_dirs(tree, "C:\\repo"), {"rs/moq-cli", "rs/moq-lite"}) + self.check("C:\\repo", "\\") + + def test_parentheses(self): + self.check("/repo (copy)", "/") if __name__ == "__main__":