diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..5f2de312c --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,528 @@ +name: Benchmark + +on: + issue_comment: + types: [created] + push: + branches: [master] + paths: + - "src/**" + - "benchmarks/**" + - "requirements/benchmark.txt" + - "setup.py" + - "pyproject.toml" + - ".github/workflows/benchmark.yml" + release: + types: [published] + workflow_dispatch: + inputs: + filter: + description: "pytest -k expression (empty runs all benchmarks)" + required: false + default: "" + timelimit: + description: "Seconds per benchmark run (Valgrind runs get 3x)" + required: false + default: "600" + +permissions: + contents: read + +concurrency: + group: ${{ github.event_name == 'issue_comment' && format('benchmark-{0}', github.run_id) || format('benchmark-{0}', github.ref) }} + cancel-in-progress: ${{ github.event_name != 'issue_comment' }} + +jobs: + start: + if: >- + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/benchmark') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + issues: write + pull-requests: write + outputs: + comment_id: ${{ steps.post.outputs.comment_id }} + request: ${{ steps.post.outputs.request }} + steps: + - name: Post status comment + id: post + env: + GH_TOKEN: ${{ github.token }} + COMMENT: ${{ github.event.comment.body }} + COMMENT_URL: ${{ github.event.comment.html_url }} + REQUESTER: ${{ github.event.comment.user.login }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + gh api "repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions" -f content=eyes > /dev/null + command=$(printf '%s\n' "$COMMENT" | head -n1 | tr -cd 'A-Za-z0-9_/= ' | cut -c1-80) + request="[\`$command\`]($COMMENT_URL) by @$REQUESTER" + body="⏳ $request: building. Next update at $(date -u -d '+5 min' +%H:%M) UTC. [Run]($RUN_URL)" + id=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments" -f body="$body" --jq .id) + echo "comment_id=$id" >> "$GITHUB_OUTPUT" + echo "request=$request" >> "$GITHUB_OUTPUT" + + prepare: + if: >- + github.event_name != 'issue_comment' || + (github.event.issue.pull_request && + startsWith(github.event.comment.body, '/benchmark') && + contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association)) + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + warm: ${{ steps.refs.outputs.warm }} + head: ${{ steps.refs.outputs.head }} + base: ${{ steps.refs.outputs.base }} + behind: ${{ steps.refs.outputs.behind }} + release: ${{ steps.refs.outputs.release }} + builds: ${{ steps.refs.outputs.builds }} + filter: ${{ steps.refs.outputs.filter }} + timelimit: ${{ steps.refs.outputs.timelimit }} + job_minutes: ${{ steps.refs.outputs.job_minutes }} + steps: + - name: Resolve commits and options + id: refs + env: + GH_TOKEN: ${{ github.token }} + COMMENT: ${{ github.event.comment.body }} + INPUT_FILTER: ${{ inputs.filter }} + INPUT_TIMELIMIT: ${{ inputs.timelimit }} + run: | + repo=${{ github.repository }} + warm=false + case "${{ github.event_name }}" in + issue_comment) + pr=$(gh api "repos/$repo/pulls/${{ github.event.issue.number }}") + head=$(jq -r .head.sha <<< "$pr") + base=$(gh api "repos/$repo/branches/$(jq -r .base.ref <<< "$pr")" --jq .commit.sha) + args=$(printf '%s\n' "$COMMENT" | head -n1 | sed -E 's|^/benchmark[[:space:]]*||') + ;; + workflow_dispatch) + head=${{ github.sha }} + base=$(gh api "repos/$repo/branches/${{ github.event.repository.default_branch }}" --jq .commit.sha) + args="$INPUT_FILTER timelimit=${INPUT_TIMELIMIT:-600}" + ;; + *) + warm=true + base=$(gh api "repos/$repo/branches/${{ github.event.repository.default_branch }}" --jq .commit.sha) + head=$base + args="" + ;; + esac + timelimit=600 + filter="" + for word in $args; do + if [[ "$(tr 'A-Z' 'a-z' <<< "$word")" =~ ^timelim(it)?=([0-9]+)$ ]]; then + timelimit=${BASH_REMATCH[2]} + else + filter="$filter $word" + fi + done + filter=${filter# } + if [ "$timelimit" -lt 1 ] || [ "$timelimit" -gt 6600 ]; then + echo "::error::timelimit must be between 1 and 6600 seconds" + exit 1 + fi + if [[ ! "$filter" =~ ^[A-Za-z0-9_\ ]*$ ]]; then + echo "::error::The filter may only contain letters, digits, '_' and spaces" + exit 1 + fi + release=$(gh api repos/scipopt/PySCIPOpt/releases/latest --jq .tag_name) + builds=$(jq -nc --arg repo "$repo" --arg base "$base" --arg head "$head" --arg release "$release" --argjson warm "$warm" ' + [{side: "base", repository: $repo, ref: $base, list: $warm}] + + (if $warm then [] else [{side: "pr", repository: $repo, ref: $head, list: true}] end) + + [{side: "release", repository: "scipopt/PySCIPOpt", ref: $release, list: false}]') + { + echo "warm=$warm" + echo "head=$head" + echo "base=$base" + echo "behind=$(gh api "repos/$repo/compare/$base...$head" --jq .behind_by)" + echo "release=$release" + echo "builds=$builds" + echo "filter=$filter" + echo "timelimit=$timelimit" + echo "job_minutes=$(( (3 * timelimit + 59) / 60 + 30 ))" + } >> "$GITHUB_OUTPUT" + + build: + needs: prepare + name: "build: ${{ matrix.side }}" + runs-on: ubuntu-22.04 + timeout-minutes: 30 + strategy: + matrix: + include: ${{ fromJSON(needs.prepare.outputs.builds) }} + outputs: + matrix: ${{ steps.list.outputs.matrix }} + steps: + - uses: actions/cache@v4 + id: cache + if: matrix.side != 'pr' + with: + path: wheels/${{ matrix.side }} + key: wheel-${{ matrix.ref }}-py3.13 + + - uses: actions/checkout@v4 + with: + repository: ${{ matrix.repository }} + ref: ${{ matrix.ref }} + path: src + persist-credentials: false + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Build wheel against the SCIP in pyproject.toml + if: steps.cache.outputs.cache-hit != 'true' + run: | + url=$(grep -o 'https://github.com/scipopt/scipoptsuite-deploy/releases/download/v[0-9.]*' src/pyproject.toml | head -1) + wget --quiet --tries=5 "$url/libscip-linux.zip" -O scip.zip + unzip -q scip.zip + SCIPOPTDIR="$PWD/scip_install" python -m pip wheel --no-deps -w dist ./src + sudo apt-get update && sudo apt-get install -y libgfortran5 + python -m pip install -q auditwheel patchelf + out="wheels/${{ matrix.side }}" + LD_LIBRARY_PATH="$PWD/scip_install/lib" auditwheel repair --plat "manylinux_2_35_$(uname -m)" -w "$out" dist/*.whl + version=$(awk '/#define SCIP_VERSION_(MAJOR|MINOR|PATCH)/ {print $3}' scip_install/include/scip/config.h | paste -sd.) + echo "SCIP $version (deploy ${url##*/})" > "$out/scip.txt" + + - name: List benchmarks + id: list + if: matrix.list + env: + FILTER: ${{ needs.prepare.outputs.filter }} + run: | + python -m venv venv + venv/bin/pip install -q -r src/requirements/benchmark.txt wheels/${{ matrix.side }}/*.whl + select=() + [ -n "$FILTER" ] && select=(-k "$FILTER") + cd src + for tier in "not timing" "timing"; do + ../venv/bin/python -m pytest benchmarks --collect-only -q -m "$tier" "${select[@]}" \ + | { grep '::' || true; } \ + | jq -R --arg tier "$tier" '{id: ., name: (split("::") | last), timing: ($tier == "timing")}' + done | jq -sc . > ../matrix.json + if [ "$(cat ../matrix.json)" = "[]" ]; then + echo "::error::No benchmark matches the filter" + exit 1 + fi + echo "matrix=$(cat ../matrix.json)" >> "$GITHUB_OUTPUT" + + - uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.side }} + path: wheels/ + + bench: + needs: [prepare, build] + name: "bench: ${{ matrix.name }}" + runs-on: ubuntu-22.04 + timeout-minutes: ${{ fromJSON(needs.prepare.outputs.job_minutes) }} + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.build.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.prepare.outputs.head }} + path: pr + persist-credentials: false + + - uses: actions/download-artifact@v4 + with: + pattern: wheels-* + path: wheels + merge-multiple: true + + - name: Install Valgrind + run: sudo apt-get update && sudo apt-get install -y valgrind + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install wheels + id: install + run: | + mkdir -p results + for side in base pr release; do + [ -d "wheels/$side" ] || continue + python -m venv "venv-$side" + "venv-$side/bin/pip" install -q -r pr/requirements/benchmark.txt "wheels/$side"/*.whl + cp "wheels/$side/scip.txt" "results/scip-$side.txt" + done + cc -O2 -shared -fPIC -o callgrind.so pr/benchmarks/callgrind.c + deps=$(venv-base/bin/pip freeze | grep -iv '^pyscipopt' | sha256sum | cut -c1-16) + echo "cache_env=$ImageOS-$ImageVersion-$deps-${{ hashFiles('pr/benchmarks/**') }}" >> "$GITHUB_OUTPUT" + + - uses: actions/cache@v4 + with: + path: cached/base + key: results-${{ needs.prepare.outputs.base }}-${{ steps.install.outputs.cache_env }}-${{ matrix.id }} + + - uses: actions/cache@v4 + with: + path: cached/release + key: results-${{ needs.prepare.outputs.release }}-${{ steps.install.outputs.cache_env }}-${{ matrix.id }} + + - name: Measure + env: + BENCH_ID: ${{ matrix.id }} + TIMING: ${{ matrix.timing }} + INDEX: ${{ strategy.job-index }} + TIMELIMIT: ${{ needs.prepare.outputs.timelimit }} + WARM: ${{ needs.prepare.outputs.warm }} + BENCH_CALLGRIND_LIB: ${{ github.workspace }}/callgrind.so + PYTHONHASHSEED: "0" + run: | + mkdir -p results + cd pr + run() { + local side=$1 mode=$2 out=$3 limit="${TIMELIMIT}s" code=0 + shift 3 + [ "$mode" = instructions ] && limit="$(( 3 * TIMELIMIT ))s" + BENCH_MODE=$mode BENCH_OUT="$out" \ + timeout "$limit" "$@" "../venv-$side/bin/python" -m pytest "$BENCH_ID" -q -p no:cacheprovider || code=$? + if [ "$code" -ne 0 ]; then + local why="failed ($mode)" + [ "$code" -eq 124 ] && why="timed out after $limit ($mode)" + echo "::error::$BENCH_ID $why on $side" + jq -nc --arg name "$BENCH_ID" --arg error "$why" '{name: $name, error: $error}' >> "$out" + return 1 + fi + } + deterministic() { + local side=$1 out=$2 + run "$side" memory "$out" || return 1 + if [ "$TIMING" != "true" ]; then + mkdir -p "../callgrind-$side" + BENCH_CALLGRIND_OUT="../callgrind-$side/out" run "$side" instructions "$out" \ + setarch -R valgrind -q --tool=callgrind --instr-atstart=no --callgrind-out-file="../callgrind-$side/out" || return 1 + fi + } + + declare -A ok=([base]=true [release]=true) + for side in base release; do + cached="../cached/$side/results.jsonl" + if [ -s "$cached" ]; then + echo "Using cached $side results" + elif deterministic "$side" "../$side.tmp"; then + mkdir -p "../cached/$side" + mv "../$side.tmp" "$cached" + else + echo "::warning::$BENCH_ID does not run on $side" + ok[$side]=false + continue + fi + cat "$cached" >> "../results/$side-$INDEX.jsonl" + done + [ "$WARM" = false ] || exit 0 + + deterministic pr "../results/pr-$INDEX.jsonl" || exit 1 + if [ "$TIMING" = "true" ]; then + [ "${ok[base]}" = false ] || run base time "../results/base-$INDEX.jsonl" || true + run pr time "../results/pr-$INDEX.jsonl" + [ "${ok[release]}" = false ] || run release time "../results/release-$INDEX.jsonl" || true + fi + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: results-${{ strategy.job-index }} + path: results/ + + progress: + needs: start + runs-on: ubuntu-latest + timeout-minutes: 360 + permissions: + actions: read + pull-requests: write + steps: + - name: Update status comment + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + COMMENT_ID: ${{ needs.start.outputs.comment_id }} + REQUEST: ${{ needs.start.outputs.request }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + shell: python + run: | + import json, os, subprocess, time + from datetime import datetime, timedelta, timezone + + INTERVAL = 300 + env = os.environ + + def gh(*args): + return subprocess.run(["gh", "api", *args], check=True, capture_output=True, text=True).stdout + + def jobs(run_id): + out = gh("--paginate", f"repos/{env['REPO']}/actions/runs/{run_id}/jobs?per_page=100", "--jq", ".jobs[]") + return [json.loads(line) for line in out.splitlines()] + + def parse(ts): + return datetime.fromisoformat(ts.replace("Z", "+00:00")) + + def minutes(seconds): + return f"{max(1, round(seconds / 60))} min" + + def durations(jobs): + return { + j["name"]: (parse(j["completed_at"]) - parse(j["started_at"])).total_seconds() + for j in jobs + if j["name"].startswith("bench: ") and j["conclusion"] == "success" + } + + def previous_durations(): + workflow = gh(f"repos/{env['REPO']}/actions/runs/{env['RUN_ID']}", "--jq", ".workflow_id").strip() + runs = gh(f"repos/{env['REPO']}/actions/workflows/{workflow}/runs?status=success&per_page=5", + "--jq", ".workflow_runs[].id").split() + for run_id in runs: + if known := durations(jobs(run_id)): + return known + return {} + + def seconds_left(bench, now, expected): + expected = {**expected, **durations(bench)} + if not expected: + return None + left = [0] + for j in bench: + total = expected.get(j["name"], max(expected.values())) + if j["status"] == "in_progress": + left.append(total - (now - parse(j["started_at"])).total_seconds()) + elif j["status"] != "completed": + left.append(total) + return max(left) + + def build_text(current): + parts = [] + for j in current: + if not j["name"].startswith("build: "): + continue + step = next((s["name"] for s in j.get("steps", []) if s["status"] == "in_progress"), None) + state = "done" if j["status"] == "completed" else step or "queued" + parts.append(f"{j['name'].removeprefix('build: ')}: {state}") + return "building (" + ", ".join(parts) + ")" if parts else "building" + + start = datetime.now(timezone.utc) + expected = previous_durations() + while True: + time.sleep(INTERVAL) + current = jobs(env["RUN_ID"]) + setup = [j for j in current if j["name"] == "prepare" or j["name"].startswith("build: ")] + bench = [j for j in current if j["name"].startswith("bench: ")] + if any(j["conclusion"] not in (None, "success") for j in setup): + break + if bench and all(j["status"] == "completed" for j in bench): + break + now = datetime.now(timezone.utc) + if bench: + done = sum(j["status"] == "completed" for j in bench) + text = f"{done}/{len(bench)} benchmarks done" + left = seconds_left(bench, now, expected) + if left is not None: + text += f", about {minutes(left)} left" if left > 30 else ", finishing up" + else: + text = build_text(current) + body = ( + f"⏳ {env['REQUEST']}: {text}. Running for {minutes((now - start).total_seconds())}, " + f"next update at {now + timedelta(seconds=INTERVAL):%H:%M} UTC. [Run]({env['RUN_URL']})" + ) + gh("-X", "PATCH", f"repos/{env['REPO']}/issues/comments/{env['COMMENT_ID']}", "-f", f"body={body}") + + report: + needs: [prepare, build, bench] + if: always() && needs.build.result == 'success' && needs.prepare.outputs.warm == 'false' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + path: tools + persist-credentials: false + + - uses: actions/download-artifact@v4 + with: + pattern: results-* + path: results + merge-multiple: true + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Compare + env: + BENCH_RESULT: ${{ needs.bench.result }} + BEHIND: ${{ needs.prepare.outputs.behind }} + RELEASE: ${{ needs.prepare.outputs.release }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + scip() { cat "results/scip-$1.txt" 2>/dev/null || echo "unknown SCIP"; } + mismatch() { + [ "$(scip "$1")" = "$(scip pr)" ] || printf '\n%s\n' "$2 uses $(scip "$1"), the PR uses $(scip pr), so the differences include SCIP changes." + } + { + python tools/benchmarks/compare.py --base results/base-*.jsonl --pr results/pr-*.jsonl + mismatch base Base + if compgen -G "results/release-*.jsonl" > /dev/null; then + echo + echo "
Against $RELEASE" + echo + python tools/benchmarks/compare.py --title "$RELEASE → PR" --base results/release-*.jsonl --pr results/pr-*.jsonl + mismatch release "$RELEASE" + echo + echo "
" + fi + echo + [ "$BENCH_RESULT" = success ] || echo "Some benchmark jobs failed; see the run." + [ "$BEHIND" = 0 ] || echo "The PR is $BEHIND commits behind its base branch, so the comparison includes those commits. Merge or rebase first for a clean result." + echo "Base \`${{ needs.prepare.outputs.base }}\`, PR \`${{ needs.prepare.outputs.head }}\`. [Run]($RUN_URL)" + } > comment.md + cat comment.md >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@v4 + with: + name: comment + path: comment.md + + comment: + needs: [start, prepare, build, bench, report] + if: always() && needs.start.outputs.comment_id != '' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + pull-requests: write + steps: + - uses: actions/download-artifact@v4 + if: needs.report.result == 'success' + with: + name: comment + + - name: Post results + env: + GH_TOKEN: ${{ github.token }} + RESULT: ${{ needs.report.result }} + REQUEST: ${{ needs.start.outputs.request }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + if [ "$RESULT" = success ]; then + { echo "Results for $REQUEST"; echo; cat comment.md; } > body.md + url=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments" -F body=@body.md --jq .html_url) + status="✅ $REQUEST: [results]($url)" + else + status="❌ $REQUEST: failed, see the [run]($RUN_URL)" + fi + gh api -X PATCH "repos/${{ github.repository }}/issues/comments/${{ needs.start.outputs.comment_id }}" -f body="$status" diff --git a/benchmarks/callgrind.c b/benchmarks/callgrind.c new file mode 100644 index 000000000..b70a6855d --- /dev/null +++ b/benchmarks/callgrind.c @@ -0,0 +1,13 @@ +#include + +void bench_start(void) +{ + CALLGRIND_START_INSTRUMENTATION; + CALLGRIND_ZERO_STATS; +} + +void bench_stop(void) +{ + CALLGRIND_STOP_INSTRUMENTATION; + CALLGRIND_DUMP_STATS; +} diff --git a/benchmarks/compare.py b/benchmarks/compare.py new file mode 100644 index 000000000..a9fa2c5a5 --- /dev/null +++ b/benchmarks/compare.py @@ -0,0 +1,90 @@ +import argparse +import json + +METRICS = ( + ("Instructions", "instructions", "count", 0.01), + ("Time", "seconds", "time", 0.20), + ("Peak memory", "peak_bytes", "bytes", 0.05), + ("Retained memory", "retained_bytes", "bytes", 0.05), +) +MIN_BYTES = 100_000 + + +def load(paths): + results = {} + for path in paths: + with open(path) as f: + for line in f: + record = json.loads(line) + results.setdefault(record.pop("name"), {}).update(record) + return results + + +def fmt(value, kind): + if value is None: + return "-" + if kind == "time": + for unit, scale in (("s", 1), ("ms", 1e-3), ("µs", 1e-6)): + if value >= scale: + return f"{value / scale:.2f} {unit}" + return f"{value / 1e-9:.0f} ns" + units = (("G", 1e9), ("M", 1e6), ("k", 1e3)) + suffix = "B" if kind == "bytes" else "" + digits = 1 if kind == "bytes" else 3 + for unit, scale in units: + if value >= scale: + return f"{value / scale:.{digits}f} {unit}{suffix}" + return f"{value} {suffix}".strip() + + +def cell(b, p, kind, threshold): + if b is None and p is None: + return "", False + text = f"{fmt(b, kind)} → {fmt(p, kind)}" + if b is None or p is None or b == 0: + return text, False + change = p / b - 1 + if kind == "bytes" and abs(p - b) < MIN_BYTES: + return f"{text} ({change:+.1%})", False + mark = "" + if change > threshold: + mark = " :red_circle:" + elif change < -threshold: + mark = " :green_circle:" + return f"{text} ({change:+.1%}){mark}", change > threshold + + +def main(base_paths, pr_paths, title): + base, pr = load(base_paths), load(pr_paths) + rows, regressions = [], 0 + for name in sorted(base.keys() | pr.keys()): + error = pr.get(name, {}).get("error") + if error: + rows.append(f"| `{name.split('::')[-1]}` | :x: {error} on PR |" + " |" * (len(METRICS) - 1)) + regressions += 1 + continue + cells = [] + for _, key, kind, threshold in METRICS: + text, worse = cell(base.get(name, {}).get(key), pr.get(name, {}).get(key), kind, threshold) + cells.append(text) + regressions += worse + rows.append(f"| `{name.split('::')[-1]}` | " + " | ".join(cells) + " |") + + print(f"## {title}\n") + print(f":warning: {regressions} regression{'s' * (regressions > 1)}\n" if regressions else "No regressions\n") + print("| Benchmark | " + " | ".join(m[0] for m in METRICS) + " |") + print("|---|" + "---|" * len(METRICS)) + print("\n".join(rows)) + print( + "\nInstructions from callgrind, time is the best of several runs, memory is Python-side (tracemalloc). " + "Flagged above " + ", ".join(f"{t:.0%} {label.lower()}" for label, _, _, t in METRICS) + "." + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--base", nargs="+", required=True) + parser.add_argument("--pr", nargs="+", required=True) + parser.add_argument("--title", default="Benchmark results (base → PR)") + args = parser.parse_args() + main(args.base, args.pr, args.title) diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py new file mode 100644 index 000000000..16e274e99 --- /dev/null +++ b/benchmarks/conftest.py @@ -0,0 +1,93 @@ +import ctypes +import gc +import json +import os +import time +import tracemalloc + +import pytest + +MODE = os.environ.get("BENCH_MODE", "time") +OUT = os.environ.get("BENCH_OUT") +RESULTS = [] +_dumps = 0 + + +def pytest_configure(config): + config.addinivalue_line("markers", "timing: large benchmark compared by wall-clock time") + + +def _instructions(call): + global _dumps + lib = ctypes.CDLL(os.environ["BENCH_CALLGRIND_LIB"]) + gc.collect() + gc.disable() + try: + lib.bench_start() + call() + lib.bench_stop() + finally: + gc.enable() + _dumps += 1 + with open(f"{os.environ['BENCH_CALLGRIND_OUT']}.{_dumps}") as f: + for line in f: + if line.startswith("totals:"): + return {"instructions": int(line.split()[1])} + raise RuntimeError("no totals in callgrind dump") + + +def _memory(call): + gc.collect() + tracemalloc.start() + result = call() + peak = tracemalloc.get_traced_memory()[1] + del result + gc.collect() + retained = tracemalloc.get_traced_memory()[0] + tracemalloc.stop() + return {"peak_bytes": peak, "retained_bytes": retained} + + +def _time(prepare, rounds): + best = float("inf") + for _ in range(rounds): + call = prepare() + gc.collect() + gc.disable() + try: + start = time.perf_counter() + call() + best = min(best, time.perf_counter() - start) + finally: + gc.enable() + return {"seconds": best} + + +@pytest.fixture +def bench(request): + def run(target, setup=None, rounds=5): + def prepare(): + args, kwargs = setup() if setup else ((), {}) + return lambda: target(*args, **kwargs) + + if MODE == "instructions": + record = _instructions(prepare()) + elif MODE == "memory": + record = _memory(prepare()) + else: + record = _time(prepare, rounds) + record["name"] = request.node.nodeid + RESULTS.append(record) + if OUT: + with open(OUT, "a") as f: + f.write(json.dumps(record) + "\n") + + return run + + +def pytest_terminal_summary(terminalreporter): + if RESULTS: + terminalreporter.section(f"benchmarks ({MODE})") + for r in RESULTS: + values = ", ".join(f"{k}={v:.4g}" if isinstance(v, float) else f"{k}={v}" for k, v in r.items() if k != "name") + terminalreporter.write_line(f"{r['name']}: {values}") diff --git a/benchmarks/test_expr.py b/benchmarks/test_expr.py new file mode 100644 index 000000000..c7b5a944d --- /dev/null +++ b/benchmarks/test_expr.py @@ -0,0 +1,89 @@ +import numpy as np + +from pyscipopt import Model, cos, exp, log, quicksum, sin, sqrt + + +def _vars(n): + m = Model() + return m, [m.addVar() for _ in range(n)] + + +def test_quicksum(bench): + _, x = _vars(100_000) + bench(lambda: quicksum(c * v for c in range(1, 9) for v in x)) + + +def test_expr_arithmetic(bench): + _, x = _vars(8000) + + def run(): + e = 0 + for i, v in enumerate(x): + e = e + 3 * v - i + return e <= 1 + + bench(run) + + +def test_repeated_terms(bench): + _, x = _vars(100) + bench(lambda: quicksum(float(i) * x[i % 100] for i in range(1_000_000))) + + +def test_expr_ops(bench): + _, x = _vars(3000) + exprs = [x[i] + 2 * x[i + 1] - 3 for i in range(len(x) - 1)] + + def run(): + for _ in range(30): + for e, f in zip(exprs, exprs[1:]): + -e + 3.0 * e + e + f + e - f + e * f + + bench(run) + + +def test_quadratic_expr(bench): + _, x = _vars(1000) + bench(lambda: quicksum(x[i] * x[j] for i in range(len(x)) for j in range(i, len(x)))) + + +def test_nonlinear_expr(bench): + _, x = _vars(10_000) + bench(lambda: quicksum(exp(v) + sqrt(v + 1) for v in x) <= 10) + + +def test_evaluate_nonlinear(bench): + m, x = _vars(3000) + exprs = [exp(v) + log(v + 1) + sqrt(v + 2) + sin(v) + cos(v) + v**2.5 + abs(v - 1) for v in x] + sol = m.createSol() + for i, v in enumerate(x): + m.setSolVal(sol, v, i / len(x)) + + def run(): + for _ in range(100): + for e in exprs: + sol[e] + + bench(run) + + +def test_matrix_ops(bench): + n = 100 + m = Model() + x = m.addMatrixVar((n, n)) + a = np.arange(n * n, dtype=float).reshape(n, n) + + def run(): + x.sum() + x.sum(axis=0) + np.add.reduce(x) + -x + 2 * x + a @ x + return x <= 1 + + bench(run) diff --git a/benchmarks/test_model.py b/benchmarks/test_model.py new file mode 100644 index 000000000..8600e56a1 --- /dev/null +++ b/benchmarks/test_model.py @@ -0,0 +1,82 @@ +import pytest + +from pyscipopt import Model, exp, quicksum, sqrt + + +def _model_with_vars(n): + m = Model() + m.hideOutput() + return m, [m.addVar(f"x{i}") for i in range(n)] + + +def test_add_vars(bench): + def run(): + m = Model() + for i in range(100_000): + m.addVar(f"x{i}", vtype="I", lb=0, ub=10, obj=1.0) + + bench(run) + + +@pytest.mark.timing +def test_add_linear_conss(bench): + n = 2_000_000 + + def run(m, x): + for i in range(n - 1): + m.addCons(x[i] + 2 * x[i + 1] <= 10) + + bench(run, setup=lambda: (_model_with_vars(n), {})) + + +def test_add_cons_linear(bench): + n = 300_000 + + def run(m, x): + for i in range(n - 1): + m.addConsLinear([x[i], x[i + 1]], [1.0, 2.0], rhs=10) + + bench(run, setup=lambda: (_model_with_vars(n), {})) + + +def test_add_nonlinear_conss(bench): + n = 20_000 + + def run(m, x): + for i in range(n - 1): + m.addCons(exp(x[i]) + x[i] * x[i + 1] + sqrt(x[i + 1] + 1) <= 10) + + bench(run, setup=lambda: (_model_with_vars(n), {})) + + +def test_getters_after_solve(bench): + m, x = _model_with_vars(100_000) + m.setObjective(quicksum(x)) + m.optimize() + sol = m.getBestSol() + + def run(): + for _ in range(6): + for v in x: + m.getVal(v) + m.getSolVal(sol, v) + v.getLbGlobal() + v.getObj() + + bench(run) + + +def test_enum_getters(bench): + m, x = _model_with_vars(1000) + m.setObjective(quicksum(x)) + m.optimize() + sol = m.getBestSol() + + def run(): + for _ in range(1000): + m.getStage() + sol.getOrigin() + for v in x: + v.getImplType() + + bench(run) diff --git a/benchmarks/test_plugins.py b/benchmarks/test_plugins.py new file mode 100644 index 000000000..86b3ca02a --- /dev/null +++ b/benchmarks/test_plugins.py @@ -0,0 +1,65 @@ +from pyscipopt import SCIP_EVENTTYPE, SCIP_RESULT, Branchrule, Conshdlr, Eventhdlr, Model + + +class AllFixed(Conshdlr): + def __init__(self, vars): + self.vars = vars + + def _result(self): + if all(v.getLbLocal() == v.getUbLocal() for v in self.vars): + return {"result": SCIP_RESULT.CUTOFF} + return {"result": SCIP_RESULT.INFEASIBLE} + + def conscheck(self, constraints, solution, checkintegrality, checklprows, printreason, completely): + return {"result": SCIP_RESULT.FEASIBLE} + + def consenfops(self, constraints, nusefulconss, solinfeasible, objinfeasible): + return self._result() + + def consenfolp(self, constraints, nusefulconss, solinfeasible): + return self._result() + + def conslock(self, constraint, locktype, nlockspos, nlocksneg): + for v in self.vars: + self.model.addVarLocksType(v, locktype, nlockspos + nlocksneg, nlockspos + nlocksneg) + + +class FirstFree(Branchrule): + def branchexecps(self, allowaddcons): + cands, *_ = self.model.getPseudoBranchCands() + self.model.branchVar(cands[0]) + return {"result": SCIP_RESULT.BRANCHED} + + +class NodeEvents(Eventhdlr): + def eventinit(self): + self.model.catchEvent(SCIP_EVENTTYPE.NODEFOCUSED, self) + + def eventexit(self): + self.model.dropEvent(SCIP_EVENTTYPE.NODEFOCUSED, self) + + def eventexec(self, event): + event.getType() + node = self.model.getCurrentNode() + node.getNumber() + node.getDepth() + self.model.getNNodes() + + +def _plugin_model(n=30, nodes=20_000): + m = Model() + m.hideOutput() + m.setPresolve(0) + m.setHeuristics(3) + m.setSeparating(3) + m.setParam("lp/solvefreq", -1) + m.setParam("limits/nodes", nodes) + x = [m.addVar(vtype="B") for _ in range(n)] + m.includeConshdlr(AllFixed(x), "allfixed", "", chckpriority=-1, enfopriority=-1, needscons=False) + m.includeBranchrule(FirstFree(), "firstfree", "", priority=1000000, maxdepth=-1, maxbounddist=1) + m.includeEventhdlr(NodeEvents(), "nodeevents", "") + return m + + +def test_plugin_callbacks(bench): + bench(lambda m: m.optimize(), setup=lambda: ((_plugin_model(),), {}), rounds=3) diff --git a/benchmarks/test_usecases.py b/benchmarks/test_usecases.py new file mode 100644 index 000000000..f58595008 --- /dev/null +++ b/benchmarks/test_usecases.py @@ -0,0 +1,104 @@ +import random + +import pytest + +from pyscipopt import SCIP_RESULT, Model, Pricer, quicksum + + +def test_many_small_models(bench): + rng = random.Random(0) + instances = [ + [[rng.randint(1, 20) for _ in range(100)] for _ in range(50)] for _ in range(100) + ] + + def run(): + for rows in instances: + m = Model() + x = [m.addVar(vtype="B", obj=-i) for i in range(100)] + for w in rows: + m.addCons(quicksum(wi * xi for wi, xi in zip(w, x)) <= 500) + + bench(run, rounds=3) + + +def test_modify_model(bench): + rng = random.Random(0) + n = 50_000 + m = Model() + x = [m.addVar(ub=10) for _ in range(n)] + conss = [m.addCons(quicksum(rng.sample(x, 10)) <= 50) for _ in range(5000)] + objectives = [[rng.random() for _ in range(n)] for _ in range(5)] + + def run(): + for i, c in enumerate(objectives): + m.setObjective(quicksum(ci * v for ci, v in zip(c, x)), "maximize") + for v in x: + m.chgVarUb(v, 5 + i) + m.chgVarLb(v, i % 2) + for cons in conss: + m.chgRhs(cons, 50 + i) + m.chgLhs(cons, -i) + m.chgCoefLinear(cons, x[i], 2.0 + i) + extra = [m.addCons(x[j] + x[j + 1] <= 15) for j in range(0, n - 1, 10)] + for cons in extra: + m.delCons(cons) + + bench(run, rounds=3) + + +def test_resolve_cycle(bench): + rng = random.Random(0) + n = 20_000 + m = Model() + m.hideOutput() + m.setPresolve(0) + x = [m.addVar(ub=10) for _ in range(n)] + budget = m.addCons(quicksum(x) <= 100) + objectives = [[rng.random() for _ in range(n)] for _ in range(2)] + + def run(): + for i, c in enumerate(objectives): + m.freeTransform() + m.setObjective(quicksum(ci * v for ci, v in zip(c, x)), "maximize") + m.chgRhs(budget, 100 + i) + for v in x: + m.chgVarUb(v, 5 + i % 5) + m.optimize() + [m.getVal(v) for v in x] + + bench(run, rounds=3) + + +class PoolPricer(Pricer): + def __init__(self, conss, pool, batch): + self.conss, self.pool, self.batch = conss, pool, batch + + def pricerinit(self): + self.conss = [self.model.getTransformedCons(c) for c in self.conss] + self.next = 0 + + def pricerredcost(self): + [self.model.getDualsolLinear(c) for c in self.conss] + for rows in self.pool[self.next:self.next + self.batch]: + var = self.model.addVar(obj=1000.0, pricedVar=True) + for r in rows: + self.model.addConsCoeff(self.conss[r], var, 1.0) + self.next += self.batch + return {"result": SCIP_RESULT.SUCCESS} + + +def _column_generation(rows=500, columns=200_000, nnz=20, batch=1000): + rng = random.Random(0) + m = Model() + m.hideOutput() + m.setPresolve(0) + x = [m.addVar(obj=10.0) for _ in range(rows)] + conss = [m.addCons(xi >= 1, separate=False, modifiable=True) for xi in x] + pool = [rng.sample(range(rows), nnz) for _ in range(columns)] + m.includePricer(PoolPricer(conss, pool, batch), "pool", "") + return m + + +@pytest.mark.timing +def test_branch_and_price_columns(bench): + bench(lambda m: m.optimize(), setup=lambda: ((_column_generation(),), {}), rounds=3) diff --git a/requirements/benchmark.txt b/requirements/benchmark.txt new file mode 100644 index 000000000..9cdd56296 --- /dev/null +++ b/requirements/benchmark.txt @@ -0,0 +1,2 @@ +pytest>=8 +numpy