diff --git a/bench/build.slow.test.py b/bench/build.slow.test.py index 7c3cec4c307..8bdfba17644 100644 --- a/bench/build.slow.test.py +++ b/bench/build.slow.test.py @@ -15,7 +15,7 @@ class Builds(unittest.TestCase): def test_base_builds_report_errors_and_distinguish_new_targets(self): (ROOT / ".tmp").mkdir(exist_ok=True) - for base in ("broken", "existing", "new", "log-failure", "hangs"): + for base in ("broken", "existing", "glob", "new", "new-package", "log-failure", "hangs"): with self.subTest(base=base), tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory: repo = Path(directory) / "repo" repo.mkdir() @@ -74,6 +74,25 @@ def run(*args): ) (package / "Cargo.toml").write_text(manifest + target) bench.write_text('fn main() { println!("current catalog ran"); }\n') + args = ["-p", "moq-relay", "--bench", "catalog"] + if base == "glob": + # A package pattern selects whatever the base has, and its bench targets. + args = ["-p", "moq-rel*", "--bench", "catalog"] + elif base == "new-package": + # A package the base lacks: naming it would fail the base's whole build. + added = repo / "rs/moq-e2ee" + (added / "src").mkdir(parents=True) + (added / "benches").mkdir() + (added / "src/lib.rs").touch() + (added / "Cargo.toml").write_text( + '[package]\nname="moq-e2ee"\nversion="0.0.0"\nedition="2024"\n' + '[[bench]]\nname="protect"\nharness=false\n' + ) + (added / "benches/protect.rs").write_text('fn main() { println!("current protect ran"); }\n') + members = '["rs/moq-relay", "rs/moq-e2ee"]' + (repo / "Cargo.toml").write_text(f'[workspace]\nmembers={members}\nresolver="3"\n') + run("cargo", "generate-lockfile") + args = ["-p", "moq-relay", "-p", "moq-e2ee", "--bench", "catalog", "--bench", "protect"] if base == "log-failure": tools = repo / ".tmp/tools" tools.mkdir(parents=True) @@ -85,7 +104,7 @@ def run(*args): ) tee.chmod(0o755) env["PATH"] = str(tools) + os.pathsep + env["PATH"] - result = run("bash", "bench/run.sh", "--criterion", "-p", "moq-relay", "--bench", "catalog") + result = run("bash", "bench/run.sh", "--criterion", *args) (evidence,) = (repo / ".tmp/bench").glob("run.*") log = (evidence / "base-build.log").read_text() if base != "log-failure" else "" report = json.loads((evidence / "paired.json").read_text()) @@ -106,12 +125,30 @@ def run(*args): ) else: self.assertEqual(report["unavailable"], []) - self.assertEqual("base catalog ran" in result.stdout, base in ("existing", "log-failure")) + self.assertEqual( + "base catalog ran" in result.stdout, base in ("existing", "glob", "new-package", "log-failure") + ) + if base == "new-package": + self.assertIn("current protect ran", result.stdout) if base == "log-failure": self.assertIn("Failed to write base build log:", result.stderr) self.assertNotEqual((evidence / "criterion-base.targets").read_text(), "") self.assertEqual(len(run("git", "worktree", "list", "--porcelain").stdout.split("worktree ")) - 1, 1) + def test_base_rejects_a_package_it_cannot_match(self): + # A version or URL spec would match no base name, so its existing benches would pass as new. + for args in (["-p", "moq-relay@0.0.0"], ["--package=moq-relay"], ["-pmoq-relay"]): + with self.subTest(args=args): + result = subprocess.run( + ["bash", "bench/run.sh", "--criterion", *args], + cwd=ROOT, + env=dict(os.environ, BENCH_BASE="HEAD"), + text=True, + capture_output=True, + ) + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn("takes a package as -p NAME or a glob", result.stderr) + if __name__ == "__main__": unittest.main() diff --git a/bench/run.sh b/bench/run.sh index acb7751cb89..ffc66cd8dcb 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -74,14 +74,23 @@ case $MODE in CRITERION_CARGO_ARGS=() CRITERION_FILTER_ARGS=() parsing_filters=false + previous= for argument in "${@:2}"; do if [[ $argument == -- && $parsing_filters == false ]]; then parsing_filters=true elif [[ $parsing_filters == true ]]; then CRITERION_FILTER_ARGS+=("$argument") else + # The base build matches each package against the base's names, so it takes + # only a name or a glob, as a separate argument. + if [[ -n $BASE ]] && { [[ $argument == --package=* || $argument == -p?* ]] || + [[ ($previous == -p || $previous == --package) && $argument == *[@:#]* ]]; }; then + echo "--criterion takes a package as -p NAME or a glob, not $argument" >&2 + exit 2 + fi CRITERION_CARGO_ARGS+=("$argument") fi + previous=$argument done ;; --audio) @@ -416,25 +425,48 @@ criterion_build() { | [($package | split("/") | last) + "/" + .target.name, $package, .executable] | @tsv' | sort >"$out" } -# Build the base with the current cargo arguments, minus bench targets its selected packages -# do not have yet: naming one would fail the whole build. Lists nothing in OUT when every +# Build the base with the current cargo arguments, minus the packages and bench targets it +# does not have yet: naming one would fail the whole build. Lists nothing in OUT when every # selected target is new. criterion_build_base() { - local out=$1 names name index selected=0 count=${#CRITERION_CARGO_ARGS[@]} - local args=() packages=() + local out=$1 metadata package pattern names name index selected=0 asked=0 count=${#CRITERION_CARGO_ARGS[@]} + local args=() present=() packages=() : >"$out" - for ((index = 0; index + 1 < count; index++)); do - if [[ ${CRITERION_CARGO_ARGS[index]} == -p || ${CRITERION_CARGO_ARGS[index]} == --package ]]; then - packages+=("${CRITERION_CARGO_ARGS[index + 1]}") + metadata=$(cd "$WORKTREE" && cargo metadata --locked --format-version 1 --no-deps) || return 1 + mapfile -t present < <(jq -r '.packages[].name' <<<"$metadata") || return 1 + for ((index = 0; index < count; index++)); do + name=${CRITERION_CARGO_ARGS[index]} + if [[ $name == -p || $name == --package ]] && ((index + 1 < count)); then + index=$((index + 1)) + pattern=${CRITERION_CARGO_ARGS[index]} + asked=$((asked + 1)) + # A selector, literal or a pattern, that names no package of the base selects only + # new ones, whose targets are new too; cargo rejects a build that names it. + name= + for package in "${present[@]}"; do + # shellcheck disable=SC2053 # Matched as the pattern cargo reads it as. + if [[ $package == $pattern ]]; then + packages+=("$package") + name=$package + fi + done + if [[ -n $name ]]; then + args+=(--package "$pattern") + fi fi done + # Without a package left, cargo would build the whole workspace instead. + if ((asked)) && ((${#packages[@]} == 0)); then + return 0 + fi # Another package's target of the same name must not count, or cargo rejects the base build. - names=$(cd "$WORKTREE" && cargo metadata --locked --format-version 1 --no-deps | - jq -r '.packages[] | select(($ARGS.positional | length) == 0 or (.name as $name | any($ARGS.positional[]; . == $name))) - | .targets[] | select(any(.kind[]; . == "bench")) | .name' --args ${packages[@]+"${packages[@]}"}) || return 1 + names=$(jq -r '.packages[] | select(($ARGS.positional | length) == 0 or (.name as $name | any($ARGS.positional[]; . == $name))) + | .targets[] | select(any(.kind[]; . == "bench")) | .name' --args ${packages[@]+"${packages[@]}"} <<<"$metadata") || return 1 for ((index = 0; index < count; index++)); do name=${CRITERION_CARGO_ARGS[index]} - if [[ $name == --bench ]] && ((index + 1 < count)); then + if [[ $name == -p || $name == --package ]] && ((index + 1 < count)); then + index=$((index + 1)) + elif [[ $name == --bench ]] && ((index + 1 < count)); then index=$((index + 1)) name=${CRITERION_CARGO_ARGS[index]} selected=$((selected + 1))