From a49c2a85f4db08777c94f78484244de1deee63e5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 10:58:58 -0800 Subject: [PATCH 1/4] fix(bench): a paired base build skips a package the base does not have Co-Authored-By: Claude Opus 5.5 --- bench/build.slow.test.py | 26 +++++++++++++++++++++++--- bench/run.sh | 26 +++++++++++++++++++------- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/bench/build.slow.test.py b/bench/build.slow.test.py index 7c3cec4c307..4da99b8afa6 100644 --- a/bench/build.slow.test.py +++ b/bench/build.slow.test.py @@ -15,7 +15,7 @@ class Builds(unittest.TestCase): def test_base_builds_report_errors_and_distinguish_new_targets(self): (ROOT / ".tmp").mkdir(exist_ok=True) - for base in ("broken", "existing", "new", "log-failure", "hangs"): + for base in ("broken", "existing", "new", "new-package", "log-failure", "hangs"): with self.subTest(base=base), tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory: repo = Path(directory) / "repo" repo.mkdir() @@ -74,6 +74,22 @@ def run(*args): ) (package / "Cargo.toml").write_text(manifest + target) bench.write_text('fn main() { println!("current catalog ran"); }\n') + args = ["-p", "moq-relay", "--bench", "catalog"] + if base == "new-package": + # A package the base lacks: naming it would fail the base's whole build. + added = repo / "rs/moq-e2ee" + (added / "src").mkdir(parents=True) + (added / "benches").mkdir() + (added / "src/lib.rs").touch() + (added / "Cargo.toml").write_text( + '[package]\nname="moq-e2ee"\nversion="0.0.0"\nedition="2024"\n' + '[[bench]]\nname="protect"\nharness=false\n' + ) + (added / "benches/protect.rs").write_text('fn main() { println!("current protect ran"); }\n') + members = '["rs/moq-relay", "rs/moq-e2ee"]' + (repo / "Cargo.toml").write_text(f'[workspace]\nmembers={members}\nresolver="3"\n') + run("cargo", "generate-lockfile") + args = ["-p", "moq-relay", "-p", "moq-e2ee", "--bench", "catalog", "--bench", "protect"] if base == "log-failure": tools = repo / ".tmp/tools" tools.mkdir(parents=True) @@ -85,7 +101,7 @@ def run(*args): ) tee.chmod(0o755) env["PATH"] = str(tools) + os.pathsep + env["PATH"] - result = run("bash", "bench/run.sh", "--criterion", "-p", "moq-relay", "--bench", "catalog") + result = run("bash", "bench/run.sh", "--criterion", *args) (evidence,) = (repo / ".tmp/bench").glob("run.*") log = (evidence / "base-build.log").read_text() if base != "log-failure" else "" report = json.loads((evidence / "paired.json").read_text()) @@ -106,7 +122,11 @@ def run(*args): ) else: self.assertEqual(report["unavailable"], []) - self.assertEqual("base catalog ran" in result.stdout, base in ("existing", "log-failure")) + self.assertEqual( + "base catalog ran" in result.stdout, base in ("existing", "new-package", "log-failure") + ) + if base == "new-package": + self.assertIn("current protect ran", result.stdout) if base == "log-failure": self.assertIn("Failed to write base build log:", result.stderr) self.assertNotEqual((evidence / "criterion-base.targets").read_text(), "") diff --git a/bench/run.sh b/bench/run.sh index acb7751cb89..674f5154317 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -416,25 +416,33 @@ criterion_build() { | [($package | split("/") | last) + "/" + .target.name, $package, .executable] | @tsv' | sort >"$out" } -# Build the base with the current cargo arguments, minus bench targets its selected packages -# do not have yet: naming one would fail the whole build. Lists nothing in OUT when every +# Build the base with the current cargo arguments, minus the packages and bench targets it +# does not have yet: naming one would fail the whole build. Lists nothing in OUT when every # selected target is new. criterion_build_base() { - local out=$1 names name index selected=0 count=${#CRITERION_CARGO_ARGS[@]} + local out=$1 metadata present names name index selected=0 asked=0 count=${#CRITERION_CARGO_ARGS[@]} local args=() packages=() : >"$out" + metadata=$(cd "$WORKTREE" && cargo metadata --locked --format-version 1 --no-deps) || return 1 + present=$(jq -r '.packages[].name' <<<"$metadata") || return 1 for ((index = 0; index + 1 < count; index++)); do if [[ ${CRITERION_CARGO_ARGS[index]} == -p || ${CRITERION_CARGO_ARGS[index]} == --package ]]; then packages+=("${CRITERION_CARGO_ARGS[index + 1]}") fi done # Another package's target of the same name must not count, or cargo rejects the base build. - names=$(cd "$WORKTREE" && cargo metadata --locked --format-version 1 --no-deps | - jq -r '.packages[] | select(($ARGS.positional | length) == 0 or (.name as $name | any($ARGS.positional[]; . == $name))) - | .targets[] | select(any(.kind[]; . == "bench")) | .name' --args ${packages[@]+"${packages[@]}"}) || return 1 + names=$(jq -r '.packages[] | select(($ARGS.positional | length) == 0 or (.name as $name | any($ARGS.positional[]; . == $name))) + | .targets[] | select(any(.kind[]; . == "bench")) | .name' --args ${packages[@]+"${packages[@]}"} <<<"$metadata") || return 1 for ((index = 0; index < count; index++)); do name=${CRITERION_CARGO_ARGS[index]} - if [[ $name == --bench ]] && ((index + 1 < count)); then + if [[ $name == -p || $name == --package ]] && ((index + 1 < count)); then + index=$((index + 1)) + asked=$((asked + 1)) + # A package the base lacks is new, as its targets are. + if grep -qxF -- "${CRITERION_CARGO_ARGS[index]}" <<<"$present"; then + args+=("$name" "${CRITERION_CARGO_ARGS[index]}") + fi + elif [[ $name == --bench ]] && ((index + 1 < count)); then index=$((index + 1)) name=${CRITERION_CARGO_ARGS[index]} selected=$((selected + 1)) @@ -449,6 +457,10 @@ criterion_build_base() { if ((selected)) && [[ " ${args[*]} " != *" --bench "* ]]; then return 0 fi + # With no package left, cargo would build the whole workspace instead. + if ((asked)) && [[ " ${args[*]} " != *" -p "* && " ${args[*]} " != *" --package "* ]]; then + return 0 + fi criterion_build "$out" "$WORKTREE" ${args[@]+"${args[@]}"} } From efe06b3ef35f6e17d62f5d3cb23b9e18d1d5c745 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 11:02:12 -0800 Subject: [PATCH 2/4] fix(bench): a package pattern still selects what the base has Co-Authored-By: Claude Opus 5.5 --- bench/build.slow.test.py | 9 ++++++--- bench/run.sh | 5 +++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/bench/build.slow.test.py b/bench/build.slow.test.py index 4da99b8afa6..b8863740d06 100644 --- a/bench/build.slow.test.py +++ b/bench/build.slow.test.py @@ -15,7 +15,7 @@ class Builds(unittest.TestCase): def test_base_builds_report_errors_and_distinguish_new_targets(self): (ROOT / ".tmp").mkdir(exist_ok=True) - for base in ("broken", "existing", "new", "new-package", "log-failure", "hangs"): + for base in ("broken", "existing", "glob", "new", "new-package", "log-failure", "hangs"): with self.subTest(base=base), tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory: repo = Path(directory) / "repo" repo.mkdir() @@ -75,7 +75,10 @@ def run(*args): (package / "Cargo.toml").write_text(manifest + target) bench.write_text('fn main() { println!("current catalog ran"); }\n') args = ["-p", "moq-relay", "--bench", "catalog"] - if base == "new-package": + if base == "glob": + # Patterns select whatever the base has. + args = ["-p", "moq-rel*", "--bench", "*"] + elif base == "new-package": # A package the base lacks: naming it would fail the base's whole build. added = repo / "rs/moq-e2ee" (added / "src").mkdir(parents=True) @@ -123,7 +126,7 @@ def run(*args): else: self.assertEqual(report["unavailable"], []) self.assertEqual( - "base catalog ran" in result.stdout, base in ("existing", "new-package", "log-failure") + "base catalog ran" in result.stdout, base in ("existing", "glob", "new-package", "log-failure") ) if base == "new-package": self.assertIn("current protect ran", result.stdout) diff --git a/bench/run.sh b/bench/run.sh index 674f5154317..0eaa27755d5 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -438,8 +438,9 @@ criterion_build_base() { if [[ $name == -p || $name == --package ]] && ((index + 1 < count)); then index=$((index + 1)) asked=$((asked + 1)) - # A package the base lacks is new, as its targets are. - if grep -qxF -- "${CRITERION_CARGO_ARGS[index]}" <<<"$present"; then + # A package the base lacks is new, as its targets are; a glob selects whatever the + # base has. + if [[ ${CRITERION_CARGO_ARGS[index]} == *[*?[]* ]] || grep -qxF -- "${CRITERION_CARGO_ARGS[index]}" <<<"$present"; then args+=("$name" "${CRITERION_CARGO_ARGS[index]}") fi elif [[ $name == --bench ]] && ((index + 1 < count)); then From eeeedfdaf023b4d7f301c668c9ab1401957ad3c9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 11:05:07 -0800 Subject: [PATCH 3/4] fix(bench): resolve package patterns against the base's packages Co-Authored-By: Claude Opus 5.5 --- bench/build.slow.test.py | 4 ++-- bench/run.sh | 42 +++++++++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/bench/build.slow.test.py b/bench/build.slow.test.py index b8863740d06..b30d904b565 100644 --- a/bench/build.slow.test.py +++ b/bench/build.slow.test.py @@ -76,8 +76,8 @@ def run(*args): bench.write_text('fn main() { println!("current catalog ran"); }\n') args = ["-p", "moq-relay", "--bench", "catalog"] if base == "glob": - # Patterns select whatever the base has. - args = ["-p", "moq-rel*", "--bench", "*"] + # A package pattern selects whatever the base has, and its bench targets. + args = ["-p", "moq-rel*", "--bench", "catalog"] elif base == "new-package": # A package the base lacks: naming it would fail the base's whole build. added = repo / "rs/moq-e2ee" diff --git a/bench/run.sh b/bench/run.sh index 0eaa27755d5..87a991cbf60 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -420,16 +420,36 @@ criterion_build() { # does not have yet: naming one would fail the whole build. Lists nothing in OUT when every # selected target is new. criterion_build_base() { - local out=$1 metadata present names name index selected=0 asked=0 count=${#CRITERION_CARGO_ARGS[@]} - local args=() packages=() + local out=$1 metadata package pattern names name index selected=0 asked=0 count=${#CRITERION_CARGO_ARGS[@]} + local args=() present=() packages=() : >"$out" metadata=$(cd "$WORKTREE" && cargo metadata --locked --format-version 1 --no-deps) || return 1 - present=$(jq -r '.packages[].name' <<<"$metadata") || return 1 - for ((index = 0; index + 1 < count; index++)); do - if [[ ${CRITERION_CARGO_ARGS[index]} == -p || ${CRITERION_CARGO_ARGS[index]} == --package ]]; then - packages+=("${CRITERION_CARGO_ARGS[index + 1]}") + mapfile -t present < <(jq -r '.packages[].name' <<<"$metadata") || return 1 + for ((index = 0; index < count; index++)); do + name=${CRITERION_CARGO_ARGS[index]} + if [[ $name == -p || $name == --package ]] && ((index + 1 < count)); then + index=$((index + 1)) + pattern=${CRITERION_CARGO_ARGS[index]} + asked=$((asked + 1)) + # A selector, literal or a pattern, that names no package of the base selects only + # new ones, whose targets are new too; cargo rejects a build that names it. + name= + for package in "${present[@]}"; do + # shellcheck disable=SC2053 # Matched as the pattern cargo reads it as. + if [[ $package == $pattern ]]; then + packages+=("$package") + name=$package + fi + done + if [[ -n $name ]]; then + args+=(--package "$pattern") + fi fi done + # Without a package left, cargo would build the whole workspace instead. + if ((asked)) && ((${#packages[@]} == 0)); then + return 0 + fi # Another package's target of the same name must not count, or cargo rejects the base build. names=$(jq -r '.packages[] | select(($ARGS.positional | length) == 0 or (.name as $name | any($ARGS.positional[]; . == $name))) | .targets[] | select(any(.kind[]; . == "bench")) | .name' --args ${packages[@]+"${packages[@]}"} <<<"$metadata") || return 1 @@ -437,12 +457,6 @@ criterion_build_base() { name=${CRITERION_CARGO_ARGS[index]} if [[ $name == -p || $name == --package ]] && ((index + 1 < count)); then index=$((index + 1)) - asked=$((asked + 1)) - # A package the base lacks is new, as its targets are; a glob selects whatever the - # base has. - if [[ ${CRITERION_CARGO_ARGS[index]} == *[*?[]* ]] || grep -qxF -- "${CRITERION_CARGO_ARGS[index]}" <<<"$present"; then - args+=("$name" "${CRITERION_CARGO_ARGS[index]}") - fi elif [[ $name == --bench ]] && ((index + 1 < count)); then index=$((index + 1)) name=${CRITERION_CARGO_ARGS[index]} @@ -458,10 +472,6 @@ criterion_build_base() { if ((selected)) && [[ " ${args[*]} " != *" --bench "* ]]; then return 0 fi - # With no package left, cargo would build the whole workspace instead. - if ((asked)) && [[ " ${args[*]} " != *" -p "* && " ${args[*]} " != *" --package "* ]]; then - return 0 - fi criterion_build "$out" "$WORKTREE" ${args[@]+"${args[@]}"} } From 4fd481a1e9399f042b1b6524e68192c465a8e7a1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 17 Nov 2026 13:22:05 -0800 Subject: [PATCH 4/4] fix(bench): a paired run rejects a package spec the base cannot match Co-Authored-By: Claude Opus 5.5 --- bench/build.slow.test.py | 14 ++++++++++++++ bench/run.sh | 9 +++++++++ 2 files changed, 23 insertions(+) diff --git a/bench/build.slow.test.py b/bench/build.slow.test.py index b30d904b565..8bdfba17644 100644 --- a/bench/build.slow.test.py +++ b/bench/build.slow.test.py @@ -135,6 +135,20 @@ def run(*args): self.assertNotEqual((evidence / "criterion-base.targets").read_text(), "") self.assertEqual(len(run("git", "worktree", "list", "--porcelain").stdout.split("worktree ")) - 1, 1) + def test_base_rejects_a_package_it_cannot_match(self): + # A version or URL spec would match no base name, so its existing benches would pass as new. + for args in (["-p", "moq-relay@0.0.0"], ["--package=moq-relay"], ["-pmoq-relay"]): + with self.subTest(args=args): + result = subprocess.run( + ["bash", "bench/run.sh", "--criterion", *args], + cwd=ROOT, + env=dict(os.environ, BENCH_BASE="HEAD"), + text=True, + capture_output=True, + ) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn("takes a package as -p NAME or a glob", result.stderr) + if __name__ == "__main__": unittest.main() diff --git a/bench/run.sh b/bench/run.sh index 87a991cbf60..ffc66cd8dcb 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -74,14 +74,23 @@ case $MODE in CRITERION_CARGO_ARGS=() CRITERION_FILTER_ARGS=() parsing_filters=false + previous= for argument in "${@:2}"; do if [[ $argument == -- && $parsing_filters == false ]]; then parsing_filters=true elif [[ $parsing_filters == true ]]; then CRITERION_FILTER_ARGS+=("$argument") else + # The base build matches each package against the base's names, so it takes + # only a name or a glob, as a separate argument. + if [[ -n $BASE ]] && { [[ $argument == --package=* || $argument == -p?* ]] || + [[ ($previous == -p || $previous == --package) && $argument == *[@:#]* ]]; }; then + echo "--criterion takes a package as -p NAME or a glob, not $argument" >&2 + exit 2 + fi CRITERION_CARGO_ARGS+=("$argument") fi + previous=$argument done ;; --audio)