diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
new file mode 100644
index 000000000..5f2de312c
--- /dev/null
+++ b/.github/workflows/benchmark.yml
@@ -0,0 +1,528 @@
+name: Benchmark
+
+on:
+ issue_comment:
+ types: [created]
+ push:
+ branches: [master]
+ paths:
+ - "src/**"
+ - "benchmarks/**"
+ - "requirements/benchmark.txt"
+ - "setup.py"
+ - "pyproject.toml"
+ - ".github/workflows/benchmark.yml"
+ release:
+ types: [published]
+ workflow_dispatch:
+ inputs:
+ filter:
+ description: "pytest -k expression (empty runs all benchmarks)"
+ required: false
+ default: ""
+ timelimit:
+ description: "Seconds per benchmark run (Valgrind runs get 3x)"
+ required: false
+ default: "600"
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.event_name == 'issue_comment' && format('benchmark-{0}', github.run_id) || format('benchmark-{0}', github.ref) }}
+ cancel-in-progress: ${{ github.event_name != 'issue_comment' }}
+
+jobs:
+ start:
+ if: >-
+ github.event.issue.pull_request &&
+ startsWith(github.event.comment.body, '/benchmark') &&
+ contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association)
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ permissions:
+ issues: write
+ pull-requests: write
+ outputs:
+ comment_id: ${{ steps.post.outputs.comment_id }}
+ request: ${{ steps.post.outputs.request }}
+ steps:
+ - name: Post status comment
+ id: post
+ env:
+ GH_TOKEN: ${{ github.token }}
+ COMMENT: ${{ github.event.comment.body }}
+ COMMENT_URL: ${{ github.event.comment.html_url }}
+ REQUESTER: ${{ github.event.comment.user.login }}
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ run: |
+ gh api "repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions" -f content=eyes > /dev/null
+ command=$(printf '%s\n' "$COMMENT" | head -n1 | tr -cd 'A-Za-z0-9_/= ' | cut -c1-80)
+ request="[\`$command\`]($COMMENT_URL) by @$REQUESTER"
+ body="⏳ $request: building. Next update at $(date -u -d '+5 min' +%H:%M) UTC. [Run]($RUN_URL)"
+ id=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments" -f body="$body" --jq .id)
+ echo "comment_id=$id" >> "$GITHUB_OUTPUT"
+ echo "request=$request" >> "$GITHUB_OUTPUT"
+
+ prepare:
+ if: >-
+ github.event_name != 'issue_comment' ||
+ (github.event.issue.pull_request &&
+ startsWith(github.event.comment.body, '/benchmark') &&
+ contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association))
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ permissions:
+ contents: read
+ pull-requests: read
+ outputs:
+ warm: ${{ steps.refs.outputs.warm }}
+ head: ${{ steps.refs.outputs.head }}
+ base: ${{ steps.refs.outputs.base }}
+ behind: ${{ steps.refs.outputs.behind }}
+ release: ${{ steps.refs.outputs.release }}
+ builds: ${{ steps.refs.outputs.builds }}
+ filter: ${{ steps.refs.outputs.filter }}
+ timelimit: ${{ steps.refs.outputs.timelimit }}
+ job_minutes: ${{ steps.refs.outputs.job_minutes }}
+ steps:
+ - name: Resolve commits and options
+ id: refs
+ env:
+ GH_TOKEN: ${{ github.token }}
+ COMMENT: ${{ github.event.comment.body }}
+ INPUT_FILTER: ${{ inputs.filter }}
+ INPUT_TIMELIMIT: ${{ inputs.timelimit }}
+ run: |
+ repo=${{ github.repository }}
+ warm=false
+ case "${{ github.event_name }}" in
+ issue_comment)
+ pr=$(gh api "repos/$repo/pulls/${{ github.event.issue.number }}")
+ head=$(jq -r .head.sha <<< "$pr")
+ base=$(gh api "repos/$repo/branches/$(jq -r .base.ref <<< "$pr")" --jq .commit.sha)
+ args=$(printf '%s\n' "$COMMENT" | head -n1 | sed -E 's|^/benchmark[[:space:]]*||')
+ ;;
+ workflow_dispatch)
+ head=${{ github.sha }}
+ base=$(gh api "repos/$repo/branches/${{ github.event.repository.default_branch }}" --jq .commit.sha)
+ args="$INPUT_FILTER timelimit=${INPUT_TIMELIMIT:-600}"
+ ;;
+ *)
+ warm=true
+ base=$(gh api "repos/$repo/branches/${{ github.event.repository.default_branch }}" --jq .commit.sha)
+ head=$base
+ args=""
+ ;;
+ esac
+ timelimit=600
+ filter=""
+ for word in $args; do
+ if [[ "$(tr 'A-Z' 'a-z' <<< "$word")" =~ ^timelim(it)?=([0-9]+)$ ]]; then
+ timelimit=${BASH_REMATCH[2]}
+ else
+ filter="$filter $word"
+ fi
+ done
+ filter=${filter# }
+ if [ "$timelimit" -lt 1 ] || [ "$timelimit" -gt 6600 ]; then
+ echo "::error::timelimit must be between 1 and 6600 seconds"
+ exit 1
+ fi
+ if [[ ! "$filter" =~ ^[A-Za-z0-9_\ ]*$ ]]; then
+ echo "::error::The filter may only contain letters, digits, '_' and spaces"
+ exit 1
+ fi
+ release=$(gh api repos/scipopt/PySCIPOpt/releases/latest --jq .tag_name)
+ builds=$(jq -nc --arg repo "$repo" --arg base "$base" --arg head "$head" --arg release "$release" --argjson warm "$warm" '
+ [{side: "base", repository: $repo, ref: $base, list: $warm}]
+ + (if $warm then [] else [{side: "pr", repository: $repo, ref: $head, list: true}] end)
+ + [{side: "release", repository: "scipopt/PySCIPOpt", ref: $release, list: false}]')
+ {
+ echo "warm=$warm"
+ echo "head=$head"
+ echo "base=$base"
+ echo "behind=$(gh api "repos/$repo/compare/$base...$head" --jq .behind_by)"
+ echo "release=$release"
+ echo "builds=$builds"
+ echo "filter=$filter"
+ echo "timelimit=$timelimit"
+ echo "job_minutes=$(( (3 * timelimit + 59) / 60 + 30 ))"
+ } >> "$GITHUB_OUTPUT"
+
+ build:
+ needs: prepare
+ name: "build: ${{ matrix.side }}"
+ runs-on: ubuntu-22.04
+ timeout-minutes: 30
+ strategy:
+ matrix:
+ include: ${{ fromJSON(needs.prepare.outputs.builds) }}
+ outputs:
+ matrix: ${{ steps.list.outputs.matrix }}
+ steps:
+ - uses: actions/cache@v4
+ id: cache
+ if: matrix.side != 'pr'
+ with:
+ path: wheels/${{ matrix.side }}
+ key: wheel-${{ matrix.ref }}-py3.13
+
+ - uses: actions/checkout@v4
+ with:
+ repository: ${{ matrix.repository }}
+ ref: ${{ matrix.ref }}
+ path: src
+ persist-credentials: false
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.13"
+
+ - name: Build wheel against the SCIP in pyproject.toml
+ if: steps.cache.outputs.cache-hit != 'true'
+ run: |
+ url=$(grep -o 'https://github.com/scipopt/scipoptsuite-deploy/releases/download/v[0-9.]*' src/pyproject.toml | head -1)
+ wget --quiet --tries=5 "$url/libscip-linux.zip" -O scip.zip
+ unzip -q scip.zip
+ SCIPOPTDIR="$PWD/scip_install" python -m pip wheel --no-deps -w dist ./src
+ sudo apt-get update && sudo apt-get install -y libgfortran5
+ python -m pip install -q auditwheel patchelf
+ out="wheels/${{ matrix.side }}"
+ LD_LIBRARY_PATH="$PWD/scip_install/lib" auditwheel repair --plat "manylinux_2_35_$(uname -m)" -w "$out" dist/*.whl
+ version=$(awk '/#define SCIP_VERSION_(MAJOR|MINOR|PATCH)/ {print $3}' scip_install/include/scip/config.h | paste -sd.)
+ echo "SCIP $version (deploy ${url##*/})" > "$out/scip.txt"
+
+ - name: List benchmarks
+ id: list
+ if: matrix.list
+ env:
+ FILTER: ${{ needs.prepare.outputs.filter }}
+ run: |
+ python -m venv venv
+ venv/bin/pip install -q -r src/requirements/benchmark.txt wheels/${{ matrix.side }}/*.whl
+ select=()
+ [ -n "$FILTER" ] && select=(-k "$FILTER")
+ cd src
+ for tier in "not timing" "timing"; do
+ ../venv/bin/python -m pytest benchmarks --collect-only -q -m "$tier" "${select[@]}" \
+ | { grep '::' || true; } \
+ | jq -R --arg tier "$tier" '{id: ., name: (split("::") | last), timing: ($tier == "timing")}'
+ done | jq -sc . > ../matrix.json
+ if [ "$(cat ../matrix.json)" = "[]" ]; then
+ echo "::error::No benchmark matches the filter"
+ exit 1
+ fi
+ echo "matrix=$(cat ../matrix.json)" >> "$GITHUB_OUTPUT"
+
+ - uses: actions/upload-artifact@v4
+ with:
+ name: wheels-${{ matrix.side }}
+ path: wheels/
+
+ bench:
+ needs: [prepare, build]
+ name: "bench: ${{ matrix.name }}"
+ runs-on: ubuntu-22.04
+ timeout-minutes: ${{ fromJSON(needs.prepare.outputs.job_minutes) }}
+ strategy:
+ fail-fast: false
+ matrix:
+ include: ${{ fromJSON(needs.build.outputs.matrix) }}
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ needs.prepare.outputs.head }}
+ path: pr
+ persist-credentials: false
+
+ - uses: actions/download-artifact@v4
+ with:
+ pattern: wheels-*
+ path: wheels
+ merge-multiple: true
+
+ - name: Install Valgrind
+ run: sudo apt-get update && sudo apt-get install -y valgrind
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.13"
+
+ - name: Install wheels
+ id: install
+ run: |
+ mkdir -p results
+ for side in base pr release; do
+ [ -d "wheels/$side" ] || continue
+ python -m venv "venv-$side"
+ "venv-$side/bin/pip" install -q -r pr/requirements/benchmark.txt "wheels/$side"/*.whl
+ cp "wheels/$side/scip.txt" "results/scip-$side.txt"
+ done
+ cc -O2 -shared -fPIC -o callgrind.so pr/benchmarks/callgrind.c
+ deps=$(venv-base/bin/pip freeze | grep -iv '^pyscipopt' | sha256sum | cut -c1-16)
+ echo "cache_env=$ImageOS-$ImageVersion-$deps-${{ hashFiles('pr/benchmarks/**') }}" >> "$GITHUB_OUTPUT"
+
+ - uses: actions/cache@v4
+ with:
+ path: cached/base
+ key: results-${{ needs.prepare.outputs.base }}-${{ steps.install.outputs.cache_env }}-${{ matrix.id }}
+
+ - uses: actions/cache@v4
+ with:
+ path: cached/release
+ key: results-${{ needs.prepare.outputs.release }}-${{ steps.install.outputs.cache_env }}-${{ matrix.id }}
+
+ - name: Measure
+ env:
+ BENCH_ID: ${{ matrix.id }}
+ TIMING: ${{ matrix.timing }}
+ INDEX: ${{ strategy.job-index }}
+ TIMELIMIT: ${{ needs.prepare.outputs.timelimit }}
+ WARM: ${{ needs.prepare.outputs.warm }}
+ BENCH_CALLGRIND_LIB: ${{ github.workspace }}/callgrind.so
+ PYTHONHASHSEED: "0"
+ run: |
+ mkdir -p results
+ cd pr
+ run() {
+ local side=$1 mode=$2 out=$3 limit="${TIMELIMIT}s" code=0
+ shift 3
+ [ "$mode" = instructions ] && limit="$(( 3 * TIMELIMIT ))s"
+ BENCH_MODE=$mode BENCH_OUT="$out" \
+ timeout "$limit" "$@" "../venv-$side/bin/python" -m pytest "$BENCH_ID" -q -p no:cacheprovider || code=$?
+ if [ "$code" -ne 0 ]; then
+ local why="failed ($mode)"
+ [ "$code" -eq 124 ] && why="timed out after $limit ($mode)"
+ echo "::error::$BENCH_ID $why on $side"
+ jq -nc --arg name "$BENCH_ID" --arg error "$why" '{name: $name, error: $error}' >> "$out"
+ return 1
+ fi
+ }
+ deterministic() {
+ local side=$1 out=$2
+ run "$side" memory "$out" || return 1
+ if [ "$TIMING" != "true" ]; then
+ mkdir -p "../callgrind-$side"
+ BENCH_CALLGRIND_OUT="../callgrind-$side/out" run "$side" instructions "$out" \
+ setarch -R valgrind -q --tool=callgrind --instr-atstart=no --callgrind-out-file="../callgrind-$side/out" || return 1
+ fi
+ }
+
+ declare -A ok=([base]=true [release]=true)
+ for side in base release; do
+ cached="../cached/$side/results.jsonl"
+ if [ -s "$cached" ]; then
+ echo "Using cached $side results"
+ elif deterministic "$side" "../$side.tmp"; then
+ mkdir -p "../cached/$side"
+ mv "../$side.tmp" "$cached"
+ else
+ echo "::warning::$BENCH_ID does not run on $side"
+ ok[$side]=false
+ continue
+ fi
+ cat "$cached" >> "../results/$side-$INDEX.jsonl"
+ done
+ [ "$WARM" = false ] || exit 0
+
+ deterministic pr "../results/pr-$INDEX.jsonl" || exit 1
+ if [ "$TIMING" = "true" ]; then
+ [ "${ok[base]}" = false ] || run base time "../results/base-$INDEX.jsonl" || true
+ run pr time "../results/pr-$INDEX.jsonl"
+ [ "${ok[release]}" = false ] || run release time "../results/release-$INDEX.jsonl" || true
+ fi
+
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: results-${{ strategy.job-index }}
+ path: results/
+
+ progress:
+ needs: start
+ runs-on: ubuntu-latest
+ timeout-minutes: 360
+ permissions:
+ actions: read
+ pull-requests: write
+ steps:
+ - name: Update status comment
+ env:
+ GH_TOKEN: ${{ github.token }}
+ REPO: ${{ github.repository }}
+ RUN_ID: ${{ github.run_id }}
+ COMMENT_ID: ${{ needs.start.outputs.comment_id }}
+ REQUEST: ${{ needs.start.outputs.request }}
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ shell: python
+ run: |
+ import json, os, subprocess, time
+ from datetime import datetime, timedelta, timezone
+
+ INTERVAL = 300
+ env = os.environ
+
+ def gh(*args):
+ return subprocess.run(["gh", "api", *args], check=True, capture_output=True, text=True).stdout
+
+ def jobs(run_id):
+ out = gh("--paginate", f"repos/{env['REPO']}/actions/runs/{run_id}/jobs?per_page=100", "--jq", ".jobs[]")
+ return [json.loads(line) for line in out.splitlines()]
+
+ def parse(ts):
+ return datetime.fromisoformat(ts.replace("Z", "+00:00"))
+
+ def minutes(seconds):
+ return f"{max(1, round(seconds / 60))} min"
+
+ def durations(jobs):
+ return {
+ j["name"]: (parse(j["completed_at"]) - parse(j["started_at"])).total_seconds()
+ for j in jobs
+ if j["name"].startswith("bench: ") and j["conclusion"] == "success"
+ }
+
+ def previous_durations():
+ workflow = gh(f"repos/{env['REPO']}/actions/runs/{env['RUN_ID']}", "--jq", ".workflow_id").strip()
+ runs = gh(f"repos/{env['REPO']}/actions/workflows/{workflow}/runs?status=success&per_page=5",
+ "--jq", ".workflow_runs[].id").split()
+ for run_id in runs:
+ if known := durations(jobs(run_id)):
+ return known
+ return {}
+
+ def seconds_left(bench, now, expected):
+ expected = {**expected, **durations(bench)}
+ if not expected:
+ return None
+ left = [0]
+ for j in bench:
+ total = expected.get(j["name"], max(expected.values()))
+ if j["status"] == "in_progress":
+ left.append(total - (now - parse(j["started_at"])).total_seconds())
+ elif j["status"] != "completed":
+ left.append(total)
+ return max(left)
+
+ def build_text(current):
+ parts = []
+ for j in current:
+ if not j["name"].startswith("build: "):
+ continue
+ step = next((s["name"] for s in j.get("steps", []) if s["status"] == "in_progress"), None)
+ state = "done" if j["status"] == "completed" else step or "queued"
+ parts.append(f"{j['name'].removeprefix('build: ')}: {state}")
+ return "building (" + ", ".join(parts) + ")" if parts else "building"
+
+ start = datetime.now(timezone.utc)
+ expected = previous_durations()
+ while True:
+ time.sleep(INTERVAL)
+ current = jobs(env["RUN_ID"])
+ setup = [j for j in current if j["name"] == "prepare" or j["name"].startswith("build: ")]
+ bench = [j for j in current if j["name"].startswith("bench: ")]
+ if any(j["conclusion"] not in (None, "success") for j in setup):
+ break
+ if bench and all(j["status"] == "completed" for j in bench):
+ break
+ now = datetime.now(timezone.utc)
+ if bench:
+ done = sum(j["status"] == "completed" for j in bench)
+ text = f"{done}/{len(bench)} benchmarks done"
+ left = seconds_left(bench, now, expected)
+ if left is not None:
+ text += f", about {minutes(left)} left" if left > 30 else ", finishing up"
+ else:
+ text = build_text(current)
+ body = (
+ f"⏳ {env['REQUEST']}: {text}. Running for {minutes((now - start).total_seconds())}, "
+ f"next update at {now + timedelta(seconds=INTERVAL):%H:%M} UTC. [Run]({env['RUN_URL']})"
+ )
+ gh("-X", "PATCH", f"repos/{env['REPO']}/issues/comments/{env['COMMENT_ID']}", "-f", f"body={body}")
+
+ report:
+ needs: [prepare, build, bench]
+ if: always() && needs.build.result == 'success' && needs.prepare.outputs.warm == 'false'
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.repository.default_branch }}
+ path: tools
+ persist-credentials: false
+
+ - uses: actions/download-artifact@v4
+ with:
+ pattern: results-*
+ path: results
+ merge-multiple: true
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.13"
+
+ - name: Compare
+ env:
+ BENCH_RESULT: ${{ needs.bench.result }}
+ BEHIND: ${{ needs.prepare.outputs.behind }}
+ RELEASE: ${{ needs.prepare.outputs.release }}
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ run: |
+ scip() { cat "results/scip-$1.txt" 2>/dev/null || echo "unknown SCIP"; }
+ mismatch() {
+ [ "$(scip "$1")" = "$(scip pr)" ] || printf '\n%s\n' "$2 uses $(scip "$1"), the PR uses $(scip pr), so the differences include SCIP changes."
+ }
+ {
+ python tools/benchmarks/compare.py --base results/base-*.jsonl --pr results/pr-*.jsonl
+ mismatch base Base
+ if compgen -G "results/release-*.jsonl" > /dev/null; then
+ echo
+ echo "Against $RELEASE
"
+ echo
+ python tools/benchmarks/compare.py --title "$RELEASE → PR" --base results/release-*.jsonl --pr results/pr-*.jsonl
+ mismatch release "$RELEASE"
+ echo
+ echo " "
+ fi
+ echo
+ [ "$BENCH_RESULT" = success ] || echo "Some benchmark jobs failed; see the run."
+ [ "$BEHIND" = 0 ] || echo "The PR is $BEHIND commits behind its base branch, so the comparison includes those commits. Merge or rebase first for a clean result."
+ echo "Base \`${{ needs.prepare.outputs.base }}\`, PR \`${{ needs.prepare.outputs.head }}\`. [Run]($RUN_URL)"
+ } > comment.md
+ cat comment.md >> "$GITHUB_STEP_SUMMARY"
+
+ - uses: actions/upload-artifact@v4
+ with:
+ name: comment
+ path: comment.md
+
+ comment:
+ needs: [start, prepare, build, bench, report]
+ if: always() && needs.start.outputs.comment_id != ''
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ permissions:
+ pull-requests: write
+ steps:
+ - uses: actions/download-artifact@v4
+ if: needs.report.result == 'success'
+ with:
+ name: comment
+
+ - name: Post results
+ env:
+ GH_TOKEN: ${{ github.token }}
+ RESULT: ${{ needs.report.result }}
+ REQUEST: ${{ needs.start.outputs.request }}
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+ run: |
+ if [ "$RESULT" = success ]; then
+ { echo "Results for $REQUEST"; echo; cat comment.md; } > body.md
+ url=$(gh api "repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments" -F body=@body.md --jq .html_url)
+ status="✅ $REQUEST: [results]($url)"
+ else
+ status="❌ $REQUEST: failed, see the [run]($RUN_URL)"
+ fi
+ gh api -X PATCH "repos/${{ github.repository }}/issues/comments/${{ needs.start.outputs.comment_id }}" -f body="$status"
diff --git a/benchmarks/callgrind.c b/benchmarks/callgrind.c
new file mode 100644
index 000000000..b70a6855d
--- /dev/null
+++ b/benchmarks/callgrind.c
@@ -0,0 +1,13 @@
+#include
+
+void bench_start(void)
+{
+ CALLGRIND_START_INSTRUMENTATION;
+ CALLGRIND_ZERO_STATS;
+}
+
+void bench_stop(void)
+{
+ CALLGRIND_STOP_INSTRUMENTATION;
+ CALLGRIND_DUMP_STATS;
+}
diff --git a/benchmarks/compare.py b/benchmarks/compare.py
new file mode 100644
index 000000000..a9fa2c5a5
--- /dev/null
+++ b/benchmarks/compare.py
@@ -0,0 +1,90 @@
+import argparse
+import json
+
+METRICS = (
+ ("Instructions", "instructions", "count", 0.01),
+ ("Time", "seconds", "time", 0.20),
+ ("Peak memory", "peak_bytes", "bytes", 0.05),
+ ("Retained memory", "retained_bytes", "bytes", 0.05),
+)
+MIN_BYTES = 100_000
+
+
+def load(paths):
+ results = {}
+ for path in paths:
+ with open(path) as f:
+ for line in f:
+ record = json.loads(line)
+ results.setdefault(record.pop("name"), {}).update(record)
+ return results
+
+
+def fmt(value, kind):
+ if value is None:
+ return "-"
+ if kind == "time":
+ for unit, scale in (("s", 1), ("ms", 1e-3), ("µs", 1e-6)):
+ if value >= scale:
+ return f"{value / scale:.2f} {unit}"
+ return f"{value / 1e-9:.0f} ns"
+ units = (("G", 1e9), ("M", 1e6), ("k", 1e3))
+ suffix = "B" if kind == "bytes" else ""
+ digits = 1 if kind == "bytes" else 3
+ for unit, scale in units:
+ if value >= scale:
+ return f"{value / scale:.{digits}f} {unit}{suffix}"
+ return f"{value} {suffix}".strip()
+
+
+def cell(b, p, kind, threshold):
+ if b is None and p is None:
+ return "", False
+ text = f"{fmt(b, kind)} → {fmt(p, kind)}"
+ if b is None or p is None or b == 0:
+ return text, False
+ change = p / b - 1
+ if kind == "bytes" and abs(p - b) < MIN_BYTES:
+ return f"{text} ({change:+.1%})", False
+ mark = ""
+ if change > threshold:
+ mark = " :red_circle:"
+ elif change < -threshold:
+ mark = " :green_circle:"
+ return f"{text} ({change:+.1%}){mark}", change > threshold
+
+
+def main(base_paths, pr_paths, title):
+ base, pr = load(base_paths), load(pr_paths)
+ rows, regressions = [], 0
+ for name in sorted(base.keys() | pr.keys()):
+ error = pr.get(name, {}).get("error")
+ if error:
+ rows.append(f"| `{name.split('::')[-1]}` | :x: {error} on PR |" + " |" * (len(METRICS) - 1))
+ regressions += 1
+ continue
+ cells = []
+ for _, key, kind, threshold in METRICS:
+ text, worse = cell(base.get(name, {}).get(key), pr.get(name, {}).get(key), kind, threshold)
+ cells.append(text)
+ regressions += worse
+ rows.append(f"| `{name.split('::')[-1]}` | " + " | ".join(cells) + " |")
+
+ print(f"## {title}\n")
+ print(f":warning: {regressions} regression{'s' * (regressions > 1)}\n" if regressions else "No regressions\n")
+ print("| Benchmark | " + " | ".join(m[0] for m in METRICS) + " |")
+ print("|---|" + "---|" * len(METRICS))
+ print("\n".join(rows))
+ print(
+ "\nInstructions from callgrind, time is the best of several runs, memory is Python-side (tracemalloc). "
+ "Flagged above " + ", ".join(f"{t:.0%} {label.lower()}" for label, _, _, t in METRICS) + "."
+ )
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--base", nargs="+", required=True)
+ parser.add_argument("--pr", nargs="+", required=True)
+ parser.add_argument("--title", default="Benchmark results (base → PR)")
+ args = parser.parse_args()
+ main(args.base, args.pr, args.title)
diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py
new file mode 100644
index 000000000..16e274e99
--- /dev/null
+++ b/benchmarks/conftest.py
@@ -0,0 +1,93 @@
+import ctypes
+import gc
+import json
+import os
+import time
+import tracemalloc
+
+import pytest
+
+MODE = os.environ.get("BENCH_MODE", "time")
+OUT = os.environ.get("BENCH_OUT")
+RESULTS = []
+_dumps = 0
+
+
+def pytest_configure(config):
+ config.addinivalue_line("markers", "timing: large benchmark compared by wall-clock time")
+
+
+def _instructions(call):
+ global _dumps
+ lib = ctypes.CDLL(os.environ["BENCH_CALLGRIND_LIB"])
+ gc.collect()
+ gc.disable()
+ try:
+ lib.bench_start()
+ call()
+ lib.bench_stop()
+ finally:
+ gc.enable()
+ _dumps += 1
+ with open(f"{os.environ['BENCH_CALLGRIND_OUT']}.{_dumps}") as f:
+ for line in f:
+ if line.startswith("totals:"):
+ return {"instructions": int(line.split()[1])}
+ raise RuntimeError("no totals in callgrind dump")
+
+
+def _memory(call):
+ gc.collect()
+ tracemalloc.start()
+ result = call()
+ peak = tracemalloc.get_traced_memory()[1]
+ del result
+ gc.collect()
+ retained = tracemalloc.get_traced_memory()[0]
+ tracemalloc.stop()
+ return {"peak_bytes": peak, "retained_bytes": retained}
+
+
+def _time(prepare, rounds):
+ best = float("inf")
+ for _ in range(rounds):
+ call = prepare()
+ gc.collect()
+ gc.disable()
+ try:
+ start = time.perf_counter()
+ call()
+ best = min(best, time.perf_counter() - start)
+ finally:
+ gc.enable()
+ return {"seconds": best}
+
+
+@pytest.fixture
+def bench(request):
+ def run(target, setup=None, rounds=5):
+ def prepare():
+ args, kwargs = setup() if setup else ((), {})
+ return lambda: target(*args, **kwargs)
+
+ if MODE == "instructions":
+ record = _instructions(prepare())
+ elif MODE == "memory":
+ record = _memory(prepare())
+ else:
+ record = _time(prepare, rounds)
+ record["name"] = request.node.nodeid
+ RESULTS.append(record)
+ if OUT:
+ with open(OUT, "a") as f:
+ f.write(json.dumps(record) + "\n")
+
+ return run
+
+
+def pytest_terminal_summary(terminalreporter):
+ if RESULTS:
+ terminalreporter.section(f"benchmarks ({MODE})")
+ for r in RESULTS:
+ values = ", ".join(f"{k}={v:.4g}" if isinstance(v, float) else f"{k}={v}" for k, v in r.items() if k != "name")
+ terminalreporter.write_line(f"{r['name']}: {values}")
diff --git a/benchmarks/test_expr.py b/benchmarks/test_expr.py
new file mode 100644
index 000000000..c7b5a944d
--- /dev/null
+++ b/benchmarks/test_expr.py
@@ -0,0 +1,89 @@
+import numpy as np
+
+from pyscipopt import Model, cos, exp, log, quicksum, sin, sqrt
+
+
+def _vars(n):
+ m = Model()
+ return m, [m.addVar() for _ in range(n)]
+
+
+def test_quicksum(bench):
+ _, x = _vars(100_000)
+ bench(lambda: quicksum(c * v for c in range(1, 9) for v in x))
+
+
+def test_expr_arithmetic(bench):
+ _, x = _vars(8000)
+
+ def run():
+ e = 0
+ for i, v in enumerate(x):
+ e = e + 3 * v - i
+ return e <= 1
+
+ bench(run)
+
+
+def test_repeated_terms(bench):
+ _, x = _vars(100)
+ bench(lambda: quicksum(float(i) * x[i % 100] for i in range(1_000_000)))
+
+
+def test_expr_ops(bench):
+ _, x = _vars(3000)
+ exprs = [x[i] + 2 * x[i + 1] - 3 for i in range(len(x) - 1)]
+
+ def run():
+ for _ in range(30):
+ for e, f in zip(exprs, exprs[1:]):
+ -e
+ 3.0 * e
+ e + f
+ e - f
+ e * f
+
+ bench(run)
+
+
+def test_quadratic_expr(bench):
+ _, x = _vars(1000)
+ bench(lambda: quicksum(x[i] * x[j] for i in range(len(x)) for j in range(i, len(x))))
+
+
+def test_nonlinear_expr(bench):
+ _, x = _vars(10_000)
+ bench(lambda: quicksum(exp(v) + sqrt(v + 1) for v in x) <= 10)
+
+
+def test_evaluate_nonlinear(bench):
+ m, x = _vars(3000)
+ exprs = [exp(v) + log(v + 1) + sqrt(v + 2) + sin(v) + cos(v) + v**2.5 + abs(v - 1) for v in x]
+ sol = m.createSol()
+ for i, v in enumerate(x):
+ m.setSolVal(sol, v, i / len(x))
+
+ def run():
+ for _ in range(100):
+ for e in exprs:
+ sol[e]
+
+ bench(run)
+
+
+def test_matrix_ops(bench):
+ n = 100
+ m = Model()
+ x = m.addMatrixVar((n, n))
+ a = np.arange(n * n, dtype=float).reshape(n, n)
+
+ def run():
+ x.sum()
+ x.sum(axis=0)
+ np.add.reduce(x)
+ -x
+ 2 * x
+ a @ x
+ return x <= 1
+
+ bench(run)
diff --git a/benchmarks/test_model.py b/benchmarks/test_model.py
new file mode 100644
index 000000000..8600e56a1
--- /dev/null
+++ b/benchmarks/test_model.py
@@ -0,0 +1,82 @@
+import pytest
+
+from pyscipopt import Model, exp, quicksum, sqrt
+
+
+def _model_with_vars(n):
+ m = Model()
+ m.hideOutput()
+ return m, [m.addVar(f"x{i}") for i in range(n)]
+
+
+def test_add_vars(bench):
+ def run():
+ m = Model()
+ for i in range(100_000):
+ m.addVar(f"x{i}", vtype="I", lb=0, ub=10, obj=1.0)
+
+ bench(run)
+
+
+@pytest.mark.timing
+def test_add_linear_conss(bench):
+ n = 2_000_000
+
+ def run(m, x):
+ for i in range(n - 1):
+ m.addCons(x[i] + 2 * x[i + 1] <= 10)
+
+ bench(run, setup=lambda: (_model_with_vars(n), {}))
+
+
+def test_add_cons_linear(bench):
+ n = 300_000
+
+ def run(m, x):
+ for i in range(n - 1):
+ m.addConsLinear([x[i], x[i + 1]], [1.0, 2.0], rhs=10)
+
+ bench(run, setup=lambda: (_model_with_vars(n), {}))
+
+
+def test_add_nonlinear_conss(bench):
+ n = 20_000
+
+ def run(m, x):
+ for i in range(n - 1):
+ m.addCons(exp(x[i]) + x[i] * x[i + 1] + sqrt(x[i + 1] + 1) <= 10)
+
+ bench(run, setup=lambda: (_model_with_vars(n), {}))
+
+
+def test_getters_after_solve(bench):
+ m, x = _model_with_vars(100_000)
+ m.setObjective(quicksum(x))
+ m.optimize()
+ sol = m.getBestSol()
+
+ def run():
+ for _ in range(6):
+ for v in x:
+ m.getVal(v)
+ m.getSolVal(sol, v)
+ v.getLbGlobal()
+ v.getObj()
+
+ bench(run)
+
+
+def test_enum_getters(bench):
+ m, x = _model_with_vars(1000)
+ m.setObjective(quicksum(x))
+ m.optimize()
+ sol = m.getBestSol()
+
+ def run():
+ for _ in range(1000):
+ m.getStage()
+ sol.getOrigin()
+ for v in x:
+ v.getImplType()
+
+ bench(run)
diff --git a/benchmarks/test_plugins.py b/benchmarks/test_plugins.py
new file mode 100644
index 000000000..86b3ca02a
--- /dev/null
+++ b/benchmarks/test_plugins.py
@@ -0,0 +1,65 @@
+from pyscipopt import SCIP_EVENTTYPE, SCIP_RESULT, Branchrule, Conshdlr, Eventhdlr, Model
+
+
+class AllFixed(Conshdlr):
+ def __init__(self, vars):
+ self.vars = vars
+
+ def _result(self):
+ if all(v.getLbLocal() == v.getUbLocal() for v in self.vars):
+ return {"result": SCIP_RESULT.CUTOFF}
+ return {"result": SCIP_RESULT.INFEASIBLE}
+
+ def conscheck(self, constraints, solution, checkintegrality, checklprows, printreason, completely):
+ return {"result": SCIP_RESULT.FEASIBLE}
+
+ def consenfops(self, constraints, nusefulconss, solinfeasible, objinfeasible):
+ return self._result()
+
+ def consenfolp(self, constraints, nusefulconss, solinfeasible):
+ return self._result()
+
+ def conslock(self, constraint, locktype, nlockspos, nlocksneg):
+ for v in self.vars:
+ self.model.addVarLocksType(v, locktype, nlockspos + nlocksneg, nlockspos + nlocksneg)
+
+
+class FirstFree(Branchrule):
+ def branchexecps(self, allowaddcons):
+ cands, *_ = self.model.getPseudoBranchCands()
+ self.model.branchVar(cands[0])
+ return {"result": SCIP_RESULT.BRANCHED}
+
+
+class NodeEvents(Eventhdlr):
+ def eventinit(self):
+ self.model.catchEvent(SCIP_EVENTTYPE.NODEFOCUSED, self)
+
+ def eventexit(self):
+ self.model.dropEvent(SCIP_EVENTTYPE.NODEFOCUSED, self)
+
+ def eventexec(self, event):
+ event.getType()
+ node = self.model.getCurrentNode()
+ node.getNumber()
+ node.getDepth()
+ self.model.getNNodes()
+
+
+def _plugin_model(n=30, nodes=20_000):
+ m = Model()
+ m.hideOutput()
+ m.setPresolve(0)
+ m.setHeuristics(3)
+ m.setSeparating(3)
+ m.setParam("lp/solvefreq", -1)
+ m.setParam("limits/nodes", nodes)
+ x = [m.addVar(vtype="B") for _ in range(n)]
+ m.includeConshdlr(AllFixed(x), "allfixed", "", chckpriority=-1, enfopriority=-1, needscons=False)
+ m.includeBranchrule(FirstFree(), "firstfree", "", priority=1000000, maxdepth=-1, maxbounddist=1)
+ m.includeEventhdlr(NodeEvents(), "nodeevents", "")
+ return m
+
+
+def test_plugin_callbacks(bench):
+ bench(lambda m: m.optimize(), setup=lambda: ((_plugin_model(),), {}), rounds=3)
diff --git a/benchmarks/test_usecases.py b/benchmarks/test_usecases.py
new file mode 100644
index 000000000..f58595008
--- /dev/null
+++ b/benchmarks/test_usecases.py
@@ -0,0 +1,104 @@
+import random
+
+import pytest
+
+from pyscipopt import SCIP_RESULT, Model, Pricer, quicksum
+
+
+def test_many_small_models(bench):
+ rng = random.Random(0)
+ instances = [
+ [[rng.randint(1, 20) for _ in range(100)] for _ in range(50)] for _ in range(100)
+ ]
+
+ def run():
+ for rows in instances:
+ m = Model()
+ x = [m.addVar(vtype="B", obj=-i) for i in range(100)]
+ for w in rows:
+ m.addCons(quicksum(wi * xi for wi, xi in zip(w, x)) <= 500)
+
+ bench(run, rounds=3)
+
+
+def test_modify_model(bench):
+ rng = random.Random(0)
+ n = 50_000
+ m = Model()
+ x = [m.addVar(ub=10) for _ in range(n)]
+ conss = [m.addCons(quicksum(rng.sample(x, 10)) <= 50) for _ in range(5000)]
+ objectives = [[rng.random() for _ in range(n)] for _ in range(5)]
+
+ def run():
+ for i, c in enumerate(objectives):
+ m.setObjective(quicksum(ci * v for ci, v in zip(c, x)), "maximize")
+ for v in x:
+ m.chgVarUb(v, 5 + i)
+ m.chgVarLb(v, i % 2)
+ for cons in conss:
+ m.chgRhs(cons, 50 + i)
+ m.chgLhs(cons, -i)
+ m.chgCoefLinear(cons, x[i], 2.0 + i)
+ extra = [m.addCons(x[j] + x[j + 1] <= 15) for j in range(0, n - 1, 10)]
+ for cons in extra:
+ m.delCons(cons)
+
+ bench(run, rounds=3)
+
+
+def test_resolve_cycle(bench):
+ rng = random.Random(0)
+ n = 20_000
+ m = Model()
+ m.hideOutput()
+ m.setPresolve(0)
+ x = [m.addVar(ub=10) for _ in range(n)]
+ budget = m.addCons(quicksum(x) <= 100)
+ objectives = [[rng.random() for _ in range(n)] for _ in range(2)]
+
+ def run():
+ for i, c in enumerate(objectives):
+ m.freeTransform()
+ m.setObjective(quicksum(ci * v for ci, v in zip(c, x)), "maximize")
+ m.chgRhs(budget, 100 + i)
+ for v in x:
+ m.chgVarUb(v, 5 + i % 5)
+ m.optimize()
+ [m.getVal(v) for v in x]
+
+ bench(run, rounds=3)
+
+
+class PoolPricer(Pricer):
+ def __init__(self, conss, pool, batch):
+ self.conss, self.pool, self.batch = conss, pool, batch
+
+ def pricerinit(self):
+ self.conss = [self.model.getTransformedCons(c) for c in self.conss]
+ self.next = 0
+
+ def pricerredcost(self):
+ [self.model.getDualsolLinear(c) for c in self.conss]
+ for rows in self.pool[self.next:self.next + self.batch]:
+ var = self.model.addVar(obj=1000.0, pricedVar=True)
+ for r in rows:
+ self.model.addConsCoeff(self.conss[r], var, 1.0)
+ self.next += self.batch
+ return {"result": SCIP_RESULT.SUCCESS}
+
+
+def _column_generation(rows=500, columns=200_000, nnz=20, batch=1000):
+ rng = random.Random(0)
+ m = Model()
+ m.hideOutput()
+ m.setPresolve(0)
+ x = [m.addVar(obj=10.0) for _ in range(rows)]
+ conss = [m.addCons(xi >= 1, separate=False, modifiable=True) for xi in x]
+ pool = [rng.sample(range(rows), nnz) for _ in range(columns)]
+ m.includePricer(PoolPricer(conss, pool, batch), "pool", "")
+ return m
+
+
+@pytest.mark.timing
+def test_branch_and_price_columns(bench):
+ bench(lambda m: m.optimize(), setup=lambda: ((_column_generation(),), {}), rounds=3)
diff --git a/requirements/benchmark.txt b/requirements/benchmark.txt
new file mode 100644
index 000000000..9cdd56296
--- /dev/null
+++ b/requirements/benchmark.txt
@@ -0,0 +1,2 @@
+pytest>=8
+numpy