From 276a3764fb2c9d01d41b69302ee02484b157d582 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:51:57 -0800 Subject: [PATCH 1/4] chore(quest): claim quest/m2/js-bench-inventory.md Quest-Claim: c6bd8eb8d645e8fe From 4f7f15c65719fb09a63f2d5235e9f4464156de52 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:56:18 -0800 Subject: [PATCH 2/4] ci(test-inventory): flag JS bench entry points no workflow runs Every JS bench script under a bench/ directory that no other source imports or names is an entry point, and the gate now requires a CI caller for it like any standalone test script. Nightly's static loop already reaches each one through the recipe parameter it binds, so no Nightly change is needed. A browser-driving script under bench/ now counts as a test too, so a recipe that runs an unreached bench is flagged. Co-Authored-By: Claude Opus 5.5 --- quest/m2/README.md | 1 - quest/m2/js-bench-inventory.md | 13 -------- sh/gh/test-inventory.py | 52 +++++++++++++++++++++++++------ sh/gh/test-inventory.test.py | 56 ++++++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 23 deletions(-) delete mode 100644 quest/m2/js-bench-inventory.md diff --git a/quest/m2/README.md b/quest/m2/README.md index 2dfe86b62d4..c848e00b828 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -247,5 +247,4 @@ lite-07; bitrate enforcement is not part of this release. - [moq-mux's AAC config reports SBR](/quest/m2/aac-sbr-config.md) - moq-audio reads backward-compatible SBR from the config - [An ended HLS playlist has no trailing gap](/quest/m2/hls-trailing-gap.md) - no zero-duration `EXT-X-GAP` before `EXT-X-ENDLIST` - [moq-video's Apple Sink encodes on its own thread](/quest/m2/apple-sink-thread.md) - macOS and iOS ladders never encode on a tokio worker -- [The test inventory sees JS bench scripts](/quest/m2/js-bench-inventory.md) - JS bench entry points are resolved by the inventory gate - [/fetch answers from the announce sync](/quest/m2/fetch-synced.md) - a missing broadcast gets a 404 once announcements are synced, not after 30 s diff --git a/quest/m2/js-bench-inventory.md b/quest/m2/js-bench-inventory.md deleted file mode 100644 index 20d80f4ddf0..00000000000 --- a/quest/m2/js-bench-inventory.md +++ /dev/null @@ -1,13 +0,0 @@ -# [XS] The test inventory sees JS bench scripts - -## Goal - -The test-inventory gate resolves the JS bench scripts Nightly runs, so a JS bench entry point no workflow runs is flagged like any other opt-in target. - -## Plan - -Found by the agent on [moq-dev/moq#7316](https://github.com/moq-dev/moq/pull/7316): Nightly runs every JS bench through the `javascript` helper as `"$@"` (`sh/gh/nightly.sh:66-72`, the loop at `:138-155`), which the checker can't resolve, and `test-inventory.py` skips scripts under `/bench/`. Every JS bench entry point has a caller today, so this closes a coverage gap in the checker, not in CI. - -Decided 2026-10-30 (maintainer, follow-up planning after the /quest-afk run). Make the bench list something the checker can read (a static list Nightly loops over, or a recipe per bench) rather than teaching the checker shell indirection, and stop skipping `/bench/`. - -Public API: none. Wire: none. diff --git a/sh/gh/test-inventory.py b/sh/gh/test-inventory.py index 23a76348a2d..1d8d7b2b490 100644 --- a/sh/gh/test-inventory.py +++ b/sh/gh/test-inventory.py @@ -7,7 +7,9 @@ - a cargo-fuzz target or a Cargo bench target, named by some reachable run; - a `just` recipe that runs tests, reachable from a workflow; - a standalone test script (`*.test.py`, `*.test.sh`, `*.browser.ts`, or a - `*.test.ts` outside a Bun package that runs `bun test`), reachable from one. + `*.test.ts` outside a Bun package that runs `bun test`), reachable from one; +- a JS bench script under a `bench/` directory that no other source imports + or names, so it is an entry point rather than a module, reachable from one. Reachable means some workflow `run:` step leads to it through `just` recipes, their dependencies, and the scripts they name. The walk is static: a `just` @@ -37,6 +39,9 @@ import yaml SCRIPT = re.compile(r"[\w.$/{}-]*[\w-]\.(?:sh|py|ts|mts|js|mjs)\b") +JS = (".ts", ".mts", ".js", ".mjs") +# A relative module specifier in a static or dynamic import, or a re-export. +IMPORT = re.compile(r"""\b(?:from|import)\s*\(?\s*["'](\.\.?/[^"']+)["']""") # Commands that run a test suite. Python sources spell them as argument lists, # so quotes and commas are dropped before matching. RUNNER = re.compile( @@ -851,10 +856,11 @@ def recipe_line(root: Path, recipe: Recipe) -> str: return str(source.relative_to(root)) -def runs_tests(graph: Graph, recipe: Recipe, found: set[Path]) -> bool: +def runs_tests(graph: Graph, recipe: Recipe, found: set[Path], benches: set[Path]) -> bool: """A recipe runs tests when its body runs a suite, or names a script no workflow reaches - that runs one, is a test script, or drives a browser. A wrapper around a reached script, - or a bare `bun test` in a directory a reached run already tests, adds no test of its own.""" + that runs one, is a test script or a bench entry point, or drives a browser. A wrapper + around a reached script, or a bare `bun test` in a directory a reached run already tests, + adds no test of its own.""" text = recipe.text if any(recipe.cwd.is_relative_to(directory) for directory in found): text = re.sub(r"(?m)^\s*bun test\s*$", "", text) @@ -869,7 +875,7 @@ def runs_tests(graph: Graph, recipe: Recipe, found: set[Path]) -> bool: continue seen.add(path) relative = str(path.relative_to(graph.root)) - if is_test_script(relative) or drives_browser(path, relative): + if is_test_script(relative) or path in benches or drives_browser(path): return True texts.append((path.read_text(errors="replace"), path)) return False @@ -879,13 +885,34 @@ def is_test_script(relative: str) -> bool: return bool(re.search(r"\.test\.(?:py|sh|ts|js|mjs)$|\.browser\.ts$", relative)) -def drives_browser(path: Path, relative: str) -> bool: - """A script that launches a browser is a check, unless it is a benchmark.""" - if "/bench/" in f"/{relative}" or path.suffix not in (".ts", ".js", ".mjs"): +def drives_browser(path: Path) -> bool: + """A script that launches a browser is a check.""" + if path.suffix not in JS: return False return re.search(r"""from ["']playwright["']""", path.read_text(errors="replace")) is not None +def bench_entries(root: Path, graph: Graph) -> set[Path]: + """The JS bench scripts no other source imports or names: the ones something runs. + A module, or a page a bench bundles, is named by its importer; a test importing the + script it tests does not make that script a module.""" + sources = [path for path in tracked(root, *(f"*{suffix}" for suffix in JS)) + if not is_test_script(str(path.relative_to(root)))] + named = set() + for source in sources: + # Block comments go first, since a doc comment may name the script that loads this one. + text = re.sub(r"(?s)(?:^|(?<=\s))/\*.*?\*/", "", source.read_text(errors="replace")) + named.update(path for path in graph.scripts(text, source.parent, source) if path != source.resolve()) + for specifier in IMPORT.findall(text): + base = (source.parent / specifier).resolve() + named.update(path for path in (base, *(base.with_name(base.name + suffix) for suffix in JS), + *(base.with_suffix(suffix) for suffix in JS), + *(base / f"index{suffix}" for suffix in JS)) if path.is_file()) + return {path.resolve() for path in sources + if "/bench/" in f"/{path.relative_to(root)}" and not path.name.endswith(".d.ts") + and path.resolve() not in named} + + def discovered(root: Path, graph: Graph) -> set[Path]: """What a reached `bun test` discovers: the paths a root workspace's `check`, `test`, or `build` script names, or the workspace, once sh/js/workspace.py runs those scripts, and @@ -963,11 +990,12 @@ def inventory(root: Path) -> tuple[list[tuple[str, str]], list[str]]: orphans.append((f"bench {target.package}::{target.name}", where)) found = discovered(root, graph) + benches = bench_entries(root, graph) # Recipes that run tests. recipes = {recipe.path: recipe for module in justfiles.values() for recipe in module.recipes.values()} for _, recipe in sorted(recipes.items()): - if recipe.path not in graph.reached and runs_tests(graph, recipe, found): + if recipe.path not in graph.reached and runs_tests(graph, recipe, found, benches): orphans.append((f"recipe {recipe.path}", recipe_line(root, recipe))) # Standalone test scripts. @@ -978,6 +1006,12 @@ def inventory(root: Path) -> tuple[list[tuple[str, str]], list[str]]: continue if not graph.reaches(path.resolve()): orphans.append((f"script {relative}", relative)) + + # Bench entry points. + for path in sorted(benches): + if not graph.reaches(path): + relative = str(path.relative_to(root)) + orphans.append((f"script {relative}", relative)) return orphans, errors diff --git a/sh/gh/test-inventory.test.py b/sh/gh/test-inventory.test.py index d8a0baf73dc..3393e432a5a 100644 --- a/sh/gh/test-inventory.test.py +++ b/sh/gh/test-inventory.test.py @@ -262,6 +262,62 @@ def test_a_bun_workspace_covers_its_tests_once_its_runner_is_reached(self): self.write("js/net/package.json", '{"scripts": {"manual-test": "bun test src"}}') self.assertIn("script js/net/src/codec.test.ts", self.orphans()) + def test_a_js_bench_entry_point_needs_a_caller(self): + # Nightly's shape: a loop's literal names reach a recipe parameter through a helper's "$@". + self.write("sh/nightly.sh", """ + #!/usr/bin/env bash + javascript() { + local label=$1 + shift + "$@" | tee "$label.csv" + } + for benchmark in fills; do + javascript "$benchmark" just js bench "$benchmark" + done + javascript page just js page-bench + """) + self.write("js/justfile", """ + bench name: + bun net/bench/{{ name }}.ts + + page-bench: + bun watch/bench/page.ts + + gc-bench: + bun net/bench/gc.ts + """) + self.write("justfile", JUSTFILE.replace(" mod rs\n", " mod js\n mod rs\n") + " sh/nightly.sh\n") + # A module, imported with or without its extension, and a page a bench bundles are not + # entry points; a doc comment naming its loader, or a test importing it, changes nothing. + self.write("js/net/bench/fills.ts", 'import { sweep } from "./sweep";\nimport "./shared.ts";\n') + self.write("js/net/bench/sweep.ts", "/** Shared by `fills.ts` and `forward.ts`. */\n") + self.write("js/net/bench/shared.ts", "") + self.write("js/net/bench/forward.ts", 'import { sweep } from "./sweep.ts";\n') + self.write("js/net/bench/forward.test.ts", 'await import("./forward");\n') + self.write("js/net/bench/gc.ts", "") + self.write("js/watch/bench/page.ts", 'import { chromium } from "playwright";\n' + 'await Bun.build({ entrypoints: [`${import.meta.dir}/page-entry.ts`] });\n') + self.write("js/watch/bench/page-entry.ts", "") + orphans = self.orphans() + self.assertIn("script js/net/bench/forward.ts", orphans) + self.assertIn("script js/net/bench/gc.ts", orphans) + self.assertIn("recipe js::gc-bench", orphans) + for reached in ("fills", "sweep", "shared"): + self.assertNotIn(f"script js/net/bench/{reached}.ts", orphans) + self.assertNotIn("script js/watch/bench/page.ts", orphans) + self.assertNotIn("script js/watch/bench/page-entry.ts", orphans) + + # Dropping a name from the loop, or the browser bench's call, orphans its script. + nightly = (self.root / "sh/nightly.sh").read_text() + self.write("sh/nightly.sh", nightly.replace("in fills;", "in forward;") + .replace("javascript page just js page-bench\n", "")) + orphans = self.orphans() + self.assertIn("script js/net/bench/fills.ts", orphans) + self.assertNotIn("script js/net/bench/forward.ts", orphans) + self.assertIn("script js/watch/bench/page.ts", orphans) + self.assertIn("recipe js::page-bench", orphans) + self.assertNotIn("script js/watch/bench/page-entry.ts", orphans) + def test_a_selection_outside_the_ignored_set_covers_nothing(self): # Same filter, but without --run-ignored nextest skips the ignored tests. self.write("justfile", JUSTFILE.replace("--run-ignored only ", "")) From e4f7491ffbc708791829cc8f107de5fc7cc458de Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 20:53:25 -0800 Subject: [PATCH 3/4] fix(test-inventory): name micro.ts's bench entries so resume-history has a caller bench/browser/micro.ts built its non-frames entry from a template path, so the inventory could not see that bench/browser/run.sh already runs js/net/bench/resume-history.ts through it. Name every entry literally, as frames already was. resume-history's error string named micro.ts, which made micro.ts look like a module the checker never checks; reword it. Also apply ruff format to the checker and its test. Co-Authored-By: Claude Opus 5.5 --- bench/browser/micro.ts | 18 ++++++++++++------ js/net/bench/resume-history.ts | 2 +- sh/gh/test-inventory.py | 28 ++++++++++++++++++++-------- sh/gh/test-inventory.test.py | 27 +++++++++++++++++++-------- 4 files changed, 52 insertions(+), 23 deletions(-) diff --git a/bench/browser/micro.ts b/bench/browser/micro.ts index 7988efd7f79..c3459a9c10b 100644 --- a/bench/browser/micro.ts +++ b/bench/browser/micro.ts @@ -14,8 +14,16 @@ const { values, positionals } = parseArgs({ }, }); const benchmark = values.bench; -const expectedRows = ({ frames: 26, drift: 36, ledger: 12, "resume-history": 72 } as Record)[benchmark]; -if (expectedRows === undefined) throw new Error("--bench must be frames, drift, ledger, or resume-history"); +// Each entry is named in full, so sh/gh/test-inventory.py sees this harness run it. +const benchmarks: Record = { + frames: { entry: "js/net/bench/frames.ts", rows: 26 }, + drift: { entry: "js/net/bench/drift.ts", rows: 36 }, + ledger: { entry: "js/net/bench/ledger.ts", rows: 12 }, + "resume-history": { entry: "js/net/bench/resume-history.ts", rows: 72 }, +}; +const selected = Object.hasOwn(benchmarks, benchmark) ? benchmarks[benchmark] : undefined; +if (!selected) throw new Error(`--bench must be one of: ${Object.keys(benchmarks).join(", ")}`); +const expectedRows = selected.rows; const rounds = Number(values.rounds); if (!Number.isSafeInteger(rounds) || rounds < 1) throw new Error("--rounds must be a positive integer"); const roots = new Map(); @@ -27,10 +35,8 @@ for (const argument of positionals.length ? positionals : ["current=.", "aa=."]) } const scripts = new Map(); for (const [name, root] of roots) { - const entry = - benchmark === "frames" - ? resolve(root, "js/net/bench/frames.ts") - : resolve(import.meta.dir, `../../js/net/bench/${benchmark}.ts`); + // Only frames is bundled from each variant; the rest import each variant's `../src/` below. + const entry = resolve(benchmark === "frames" ? root : resolve(import.meta.dir, "../.."), selected.entry); const result = await Bun.build({ entrypoints: [entry], target: "browser", diff --git a/js/net/bench/resume-history.ts b/js/net/bench/resume-history.ts index 05e076e711e..a76023b860c 100644 --- a/js/net/bench/resume-history.ts +++ b/js/net/bench/resume-history.ts @@ -39,7 +39,7 @@ type Access = "lookup" | "fetch" | "subscribe"; /** Full collections, each from a fresh task so no frame of the caller keeps a target. */ async function collect(): Promise { const gc = (globalThis as { gc?: () => void }).gc; - if (!gc) throw new Error("resume-history needs a precise collector: run it with bench/browser/micro.ts"); + if (!gc) throw new Error("resume-history needs a precise collector: run it in Chromium through bench/browser"); for (let pass = 0; pass < 3; pass++) { await new Promise((resolve) => setTimeout(resolve, 0)); gc(); diff --git a/sh/gh/test-inventory.py b/sh/gh/test-inventory.py index 15556d7c374..6ef0c6ec775 100644 --- a/sh/gh/test-inventory.py +++ b/sh/gh/test-inventory.py @@ -957,8 +957,11 @@ def bench_entries(root: Path, graph: Graph) -> set[Path]: """The JS bench scripts no other source imports or names: the ones something runs. A module, or a page a bench bundles, is named by its importer; a test importing the script it tests does not make that script a module.""" - sources = [path for path in tracked(root, *(f"*{suffix}" for suffix in JS)) - if not is_test_script(str(path.relative_to(root)))] + sources = [ + path + for path in tracked(root, *(f"*{suffix}" for suffix in JS)) + if not is_test_script(str(path.relative_to(root))) + ] named = set() for source in sources: # Block comments go first, since a doc comment may name the script that loads this one. @@ -966,12 +969,21 @@ def bench_entries(root: Path, graph: Graph) -> set[Path]: named.update(path for path in graph.scripts(text, source.parent, source) if path != source.resolve()) for specifier in IMPORT.findall(text): base = (source.parent / specifier).resolve() - named.update(path for path in (base, *(base.with_name(base.name + suffix) for suffix in JS), - *(base.with_suffix(suffix) for suffix in JS), - *(base / f"index{suffix}" for suffix in JS)) if path.is_file()) - return {path.resolve() for path in sources - if "/bench/" in f"/{path.relative_to(root)}" and not path.name.endswith(".d.ts") - and path.resolve() not in named} + named.update( + path + for path in ( + base, + *(base.with_name(base.name + suffix) for suffix in JS), + *(base.with_suffix(suffix) for suffix in JS), + *(base / f"index{suffix}" for suffix in JS), + ) + if path.is_file() + ) + return { + path.resolve() + for path in sources + if "/bench/" in f"/{path.relative_to(root)}" and not path.name.endswith(".d.ts") and path.resolve() not in named + } def discovered(root: Path, graph: Graph) -> set[Path]: diff --git a/sh/gh/test-inventory.test.py b/sh/gh/test-inventory.test.py index 665af22be49..71232066990 100644 --- a/sh/gh/test-inventory.test.py +++ b/sh/gh/test-inventory.test.py @@ -278,7 +278,9 @@ def test_a_bun_workspace_covers_its_tests_once_its_runner_is_reached(self): def test_a_js_bench_entry_point_needs_a_caller(self): # Nightly's shape: a loop's literal names reach a recipe parameter through a helper's "$@". - self.write("sh/nightly.sh", """ + self.write( + "sh/nightly.sh", + """ #!/usr/bin/env bash javascript() { local label=$1 @@ -289,8 +291,11 @@ def test_a_js_bench_entry_point_needs_a_caller(self): javascript "$benchmark" just js bench "$benchmark" done javascript page just js page-bench - """) - self.write("js/justfile", """ + """, + ) + self.write( + "js/justfile", + """ bench name: bun net/bench/{{ name }}.ts @@ -299,7 +304,8 @@ def test_a_js_bench_entry_point_needs_a_caller(self): gc-bench: bun net/bench/gc.ts - """) + """, + ) self.write("justfile", JUSTFILE.replace(" mod rs\n", " mod js\n mod rs\n") + " sh/nightly.sh\n") # A module, imported with or without its extension, and a page a bench bundles are not # entry points; a doc comment naming its loader, or a test importing it, changes nothing. @@ -309,8 +315,11 @@ def test_a_js_bench_entry_point_needs_a_caller(self): self.write("js/net/bench/forward.ts", 'import { sweep } from "./sweep.ts";\n') self.write("js/net/bench/forward.test.ts", 'await import("./forward");\n') self.write("js/net/bench/gc.ts", "") - self.write("js/watch/bench/page.ts", 'import { chromium } from "playwright";\n' - 'await Bun.build({ entrypoints: [`${import.meta.dir}/page-entry.ts`] });\n') + self.write( + "js/watch/bench/page.ts", + 'import { chromium } from "playwright";\n' + "await Bun.build({ entrypoints: [`${import.meta.dir}/page-entry.ts`] });\n", + ) self.write("js/watch/bench/page-entry.ts", "") orphans = self.orphans() self.assertIn("script js/net/bench/forward.ts", orphans) @@ -323,8 +332,10 @@ def test_a_js_bench_entry_point_needs_a_caller(self): # Dropping a name from the loop, or the browser bench's call, orphans its script. nightly = (self.root / "sh/nightly.sh").read_text() - self.write("sh/nightly.sh", nightly.replace("in fills;", "in forward;") - .replace("javascript page just js page-bench\n", "")) + self.write( + "sh/nightly.sh", + nightly.replace("in fills;", "in forward;").replace("javascript page just js page-bench\n", ""), + ) orphans = self.orphans() self.assertIn("script js/net/bench/fills.ts", orphans) self.assertNotIn("script js/net/bench/forward.ts", orphans) From a01b87680b7caf87c79cdfe3883b527c0164b93e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 2 Nov 2026 22:34:48 -0800 Subject: [PATCH 4/4] fix(test-inventory): ignore commented-out imports when finding JS bench entry points A non-test source's line comment like `// import "./forward"` made forward.ts look like a module, so an uncalled bench dropped out of the inventory. Strip line comments before both scans, and cover the case. Co-Authored-By: Claude Opus 5.5 --- sh/gh/test-inventory.py | 4 +++- sh/gh/test-inventory.test.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sh/gh/test-inventory.py b/sh/gh/test-inventory.py index cd152efd724..c7edff90c56 100644 --- a/sh/gh/test-inventory.py +++ b/sh/gh/test-inventory.py @@ -988,8 +988,10 @@ def bench_entries(root: Path, graph: Graph) -> set[Path]: ] named = set() for source in sources: - # Block comments go first, since a doc comment may name the script that loads this one. + # Block comments go first, since a doc comment may name the script that loads this one; + # then line comments, so a commented-out import does not make its target a module. text = re.sub(r"(?s)(?:^|(?<=\s))/\*.*?\*/", "", source.read_text(errors="replace")) + text = re.sub(r"(?m)(?:^|\s)//.*$", "", text) named.update(path for path in graph.scripts(text, source.parent, source) if path != source.resolve()) for specifier in IMPORT.findall(text): base = (source.parent / specifier).resolve() diff --git a/sh/gh/test-inventory.test.py b/sh/gh/test-inventory.test.py index e8491e33e42..dce5a05e7ec 100644 --- a/sh/gh/test-inventory.test.py +++ b/sh/gh/test-inventory.test.py @@ -382,8 +382,12 @@ def test_a_js_bench_entry_point_needs_a_caller(self): ) self.write("justfile", JUSTFILE.replace(" mod rs\n", " mod js\n mod rs\n") + " sh/nightly.sh\n") # A module, imported with or without its extension, and a page a bench bundles are not - # entry points; a doc comment naming its loader, or a test importing it, changes nothing. - self.write("js/net/bench/fills.ts", 'import { sweep } from "./sweep";\nimport "./shared.ts";\n') + # entry points; a doc comment naming its loader, a commented-out import, or a test + # importing it changes nothing. + self.write( + "js/net/bench/fills.ts", + 'import { sweep } from "./sweep";\nimport "./shared.ts"; // import "./forward";\n// import "./gc";\n', + ) self.write("js/net/bench/sweep.ts", "/** Shared by `fills.ts` and `forward.ts`. */\n") self.write("js/net/bench/shared.ts", "") self.write("js/net/bench/forward.ts", 'import { sweep } from "./sweep.ts";\n')