Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 40 additions & 3 deletions bench/build.slow.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
class Builds(unittest.TestCase):
def test_base_builds_report_errors_and_distinguish_new_targets(self):
(ROOT / ".tmp").mkdir(exist_ok=True)
for base in ("broken", "existing", "new", "log-failure", "hangs"):
for base in ("broken", "existing", "glob", "new", "new-package", "log-failure", "hangs"):
with self.subTest(base=base), tempfile.TemporaryDirectory(dir=ROOT / ".tmp") as directory:
repo = Path(directory) / "repo"
repo.mkdir()
Expand Down Expand Up @@ -74,6 +74,25 @@ def run(*args):
)
(package / "Cargo.toml").write_text(manifest + target)
bench.write_text('fn main() { println!("current catalog ran"); }\n')
args = ["-p", "moq-relay", "--bench", "catalog"]
if base == "glob":
# A package pattern selects whatever the base has, and its bench targets.
args = ["-p", "moq-rel*", "--bench", "catalog"]
elif base == "new-package":
# A package the base lacks: naming it would fail the base's whole build.
added = repo / "rs/moq-e2ee"
(added / "src").mkdir(parents=True)
(added / "benches").mkdir()
(added / "src/lib.rs").touch()
(added / "Cargo.toml").write_text(
'[package]\nname="moq-e2ee"\nversion="0.0.0"\nedition="2024"\n'
'[[bench]]\nname="protect"\nharness=false\n'
)
(added / "benches/protect.rs").write_text('fn main() { println!("current protect ran"); }\n')
members = '["rs/moq-relay", "rs/moq-e2ee"]'
(repo / "Cargo.toml").write_text(f'[workspace]\nmembers={members}\nresolver="3"\n')
run("cargo", "generate-lockfile")
args = ["-p", "moq-relay", "-p", "moq-e2ee", "--bench", "catalog", "--bench", "protect"]
if base == "log-failure":
tools = repo / ".tmp/tools"
tools.mkdir(parents=True)
Expand All @@ -85,7 +104,7 @@ def run(*args):
)
tee.chmod(0o755)
env["PATH"] = str(tools) + os.pathsep + env["PATH"]
result = run("bash", "bench/run.sh", "--criterion", "-p", "moq-relay", "--bench", "catalog")
result = run("bash", "bench/run.sh", "--criterion", *args)
(evidence,) = (repo / ".tmp/bench").glob("run.*")
log = (evidence / "base-build.log").read_text() if base != "log-failure" else ""
report = json.loads((evidence / "paired.json").read_text())
Expand All @@ -106,12 +125,30 @@ def run(*args):
)
else:
self.assertEqual(report["unavailable"], [])
self.assertEqual("base catalog ran" in result.stdout, base in ("existing", "log-failure"))
self.assertEqual(
"base catalog ran" in result.stdout, base in ("existing", "glob", "new-package", "log-failure")
)
if base == "new-package":
self.assertIn("current protect ran", result.stdout)
if base == "log-failure":
self.assertIn("Failed to write base build log:", result.stderr)
self.assertNotEqual((evidence / "criterion-base.targets").read_text(), "")
self.assertEqual(len(run("git", "worktree", "list", "--porcelain").stdout.split("worktree ")) - 1, 1)

def test_base_rejects_a_package_it_cannot_match(self):
# A version or URL spec would match no base name, so its existing benches would pass as new.
for args in (["-p", "moq-relay@0.0.0"], ["--package=moq-relay"], ["-pmoq-relay"]):
with self.subTest(args=args):
result = subprocess.run(
["bash", "bench/run.sh", "--criterion", *args],
cwd=ROOT,
env=dict(os.environ, BENCH_BASE="HEAD"),
text=True,
capture_output=True,
)
self.assertEqual(result.returncode, 2, result.stdout + result.stderr)
self.assertIn("takes a package as -p NAME or a glob", result.stderr)


if __name__ == "__main__":
unittest.main()
54 changes: 43 additions & 11 deletions bench/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -74,14 +74,23 @@ case $MODE in
CRITERION_CARGO_ARGS=()
CRITERION_FILTER_ARGS=()
parsing_filters=false
previous=
for argument in "${@:2}"; do
if [[ $argument == -- && $parsing_filters == false ]]; then
parsing_filters=true
elif [[ $parsing_filters == true ]]; then
CRITERION_FILTER_ARGS+=("$argument")
else
# The base build matches each package against the base's names, so it takes
# only a name or a glob, as a separate argument.
if [[ -n $BASE ]] && { [[ $argument == --package=* || $argument == -p?* ]] ||
[[ ($previous == -p || $previous == --package) && $argument == *[@:#]* ]]; }; then
echo "--criterion takes a package as -p NAME or a glob, not $argument" >&2
exit 2
fi
CRITERION_CARGO_ARGS+=("$argument")
fi
previous=$argument
done
;;
--audio)
Expand Down Expand Up @@ -416,25 +425,48 @@ criterion_build() {
| [($package | split("/") | last) + "/" + .target.name, $package, .executable] | @tsv' | sort >"$out"
}

# Build the base with the current cargo arguments, minus bench targets its selected packages
# do not have yet: naming one would fail the whole build. Lists nothing in OUT when every
# Build the base with the current cargo arguments, minus the packages and bench targets it
# does not have yet: naming one would fail the whole build. Lists nothing in OUT when every
# selected target is new.
criterion_build_base() {
local out=$1 names name index selected=0 count=${#CRITERION_CARGO_ARGS[@]}
local args=() packages=()
local out=$1 metadata package pattern names name index selected=0 asked=0 count=${#CRITERION_CARGO_ARGS[@]}
local args=() present=() packages=()
: >"$out"
for ((index = 0; index + 1 < count; index++)); do
if [[ ${CRITERION_CARGO_ARGS[index]} == -p || ${CRITERION_CARGO_ARGS[index]} == --package ]]; then
packages+=("${CRITERION_CARGO_ARGS[index + 1]}")
metadata=$(cd "$WORKTREE" && cargo metadata --locked --format-version 1 --no-deps) || return 1
mapfile -t present < <(jq -r '.packages[].name' <<<"$metadata") || return 1
for ((index = 0; index < count; index++)); do
name=${CRITERION_CARGO_ARGS[index]}
if [[ $name == -p || $name == --package ]] && ((index + 1 < count)); then
index=$((index + 1))
pattern=${CRITERION_CARGO_ARGS[index]}
asked=$((asked + 1))
# A selector, literal or a pattern, that names no package of the base selects only
# new ones, whose targets are new too; cargo rejects a build that names it.
name=
for package in "${present[@]}"; do
# shellcheck disable=SC2053 # Matched as the pattern cargo reads it as.
if [[ $package == $pattern ]]; then
packages+=("$package")
name=$package
fi
done
if [[ -n $name ]]; then
args+=(--package "$pattern")
fi
fi
done
# Without a package left, cargo would build the whole workspace instead.
if ((asked)) && ((${#packages[@]} == 0)); then
return 0
fi
# Another package's target of the same name must not count, or cargo rejects the base build.
names=$(cd "$WORKTREE" && cargo metadata --locked --format-version 1 --no-deps |
jq -r '.packages[] | select(($ARGS.positional | length) == 0 or (.name as $name | any($ARGS.positional[]; . == $name)))
| .targets[] | select(any(.kind[]; . == "bench")) | .name' --args ${packages[@]+"${packages[@]}"}) || return 1
names=$(jq -r '.packages[] | select(($ARGS.positional | length) == 0 or (.name as $name | any($ARGS.positional[]; . == $name)))
| .targets[] | select(any(.kind[]; . == "bench")) | .name' --args ${packages[@]+"${packages[@]}"} <<<"$metadata") || return 1
for ((index = 0; index < count; index++)); do
name=${CRITERION_CARGO_ARGS[index]}
if [[ $name == --bench ]] && ((index + 1 < count)); then
if [[ $name == -p || $name == --package ]] && ((index + 1 < count)); then
index=$((index + 1))
elif [[ $name == --bench ]] && ((index + 1 < count)); then
index=$((index + 1))
name=${CRITERION_CARGO_ARGS[index]}
selected=$((selected + 1))
Expand Down