Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
528 changes: 528 additions & 0 deletions .github/workflows/benchmark.yml

Large diffs are not rendered by default.

13 changes: 13 additions & 0 deletions benchmarks/callgrind.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#include <valgrind/callgrind.h>

void bench_start(void)
{
CALLGRIND_START_INSTRUMENTATION;
CALLGRIND_ZERO_STATS;
}

void bench_stop(void)
{
CALLGRIND_STOP_INSTRUMENTATION;
CALLGRIND_DUMP_STATS;
}
90 changes: 90 additions & 0 deletions benchmarks/compare.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import argparse
import json

METRICS = (
("Instructions", "instructions", "count", 0.01),
("Time", "seconds", "time", 0.20),
("Peak memory", "peak_bytes", "bytes", 0.05),
("Retained memory", "retained_bytes", "bytes", 0.05),
)
MIN_BYTES = 100_000


def load(paths):
results = {}
for path in paths:
with open(path) as f:
for line in f:
record = json.loads(line)
results.setdefault(record.pop("name"), {}).update(record)
return results


def fmt(value, kind):
if value is None:
return "-"
if kind == "time":
for unit, scale in (("s", 1), ("ms", 1e-3), ("µs", 1e-6)):
if value >= scale:
return f"{value / scale:.2f} {unit}"
return f"{value / 1e-9:.0f} ns"
units = (("G", 1e9), ("M", 1e6), ("k", 1e3))
suffix = "B" if kind == "bytes" else ""
digits = 1 if kind == "bytes" else 3
for unit, scale in units:
if value >= scale:
return f"{value / scale:.{digits}f} {unit}{suffix}"
return f"{value} {suffix}".strip()


def cell(b, p, kind, threshold):
if b is None and p is None:
return "", False
text = f"{fmt(b, kind)} → {fmt(p, kind)}"
if b is None or p is None or b == 0:
return text, False
change = p / b - 1
if kind == "bytes" and abs(p - b) < MIN_BYTES:
return f"{text} ({change:+.1%})", False
mark = ""
if change > threshold:
mark = " :red_circle:"
elif change < -threshold:
mark = " :green_circle:"
return f"{text} ({change:+.1%}){mark}", change > threshold


def main(base_paths, pr_paths, title):
base, pr = load(base_paths), load(pr_paths)
rows, regressions = [], 0
for name in sorted(base.keys() | pr.keys()):
error = pr.get(name, {}).get("error")
if error:
rows.append(f"| `{name.split('::')[-1]}` | :x: {error} on PR |" + " |" * (len(METRICS) - 1))
regressions += 1
continue
cells = []
for _, key, kind, threshold in METRICS:
text, worse = cell(base.get(name, {}).get(key), pr.get(name, {}).get(key), kind, threshold)
cells.append(text)
regressions += worse
rows.append(f"| `{name.split('::')[-1]}` | " + " | ".join(cells) + " |")

print(f"## {title}\n")
print(f":warning: {regressions} regression{'s' * (regressions > 1)}\n" if regressions else "No regressions\n")
print("| Benchmark | " + " | ".join(m[0] for m in METRICS) + " |")
print("|---|" + "---|" * len(METRICS))
print("\n".join(rows))
print(
"\nInstructions from callgrind, time is the best of several runs, memory is Python-side (tracemalloc). "
"Flagged above " + ", ".join(f"{t:.0%} {label.lower()}" for label, _, _, t in METRICS) + "."
)


if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--base", nargs="+", required=True)
parser.add_argument("--pr", nargs="+", required=True)
parser.add_argument("--title", default="Benchmark results (base → PR)")
args = parser.parse_args()
main(args.base, args.pr, args.title)
93 changes: 93 additions & 0 deletions benchmarks/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
import ctypes
import gc
import json
import os
import time
import tracemalloc

import pytest

MODE = os.environ.get("BENCH_MODE", "time")
OUT = os.environ.get("BENCH_OUT")
RESULTS = []
_dumps = 0


def pytest_configure(config):
config.addinivalue_line("markers", "timing: large benchmark compared by wall-clock time")


def _instructions(call):
global _dumps
lib = ctypes.CDLL(os.environ["BENCH_CALLGRIND_LIB"])
gc.collect()
gc.disable()
try:
lib.bench_start()
call()
lib.bench_stop()
finally:
gc.enable()
_dumps += 1
with open(f"{os.environ['BENCH_CALLGRIND_OUT']}.{_dumps}") as f:
for line in f:
if line.startswith("totals:"):
return {"instructions": int(line.split()[1])}
raise RuntimeError("no totals in callgrind dump")


def _memory(call):
gc.collect()
tracemalloc.start()
result = call()
peak = tracemalloc.get_traced_memory()[1]
del result
gc.collect()
retained = tracemalloc.get_traced_memory()[0]
tracemalloc.stop()
return {"peak_bytes": peak, "retained_bytes": retained}


def _time(prepare, rounds):
best = float("inf")
for _ in range(rounds):
call = prepare()
gc.collect()
gc.disable()
try:
start = time.perf_counter()
call()
best = min(best, time.perf_counter() - start)
finally:
gc.enable()
return {"seconds": best}


@pytest.fixture
def bench(request):
def run(target, setup=None, rounds=5):
def prepare():
args, kwargs = setup() if setup else ((), {})
return lambda: target(*args, **kwargs)

if MODE == "instructions":
record = _instructions(prepare())
elif MODE == "memory":
record = _memory(prepare())
else:
record = _time(prepare, rounds)
record["name"] = request.node.nodeid
RESULTS.append(record)
if OUT:
with open(OUT, "a") as f:
f.write(json.dumps(record) + "\n")

return run


def pytest_terminal_summary(terminalreporter):
if RESULTS:
terminalreporter.section(f"benchmarks ({MODE})")
for r in RESULTS:
values = ", ".join(f"{k}={v:.4g}" if isinstance(v, float) else f"{k}={v}" for k, v in r.items() if k != "name")
terminalreporter.write_line(f"{r['name']}: {values}")
89 changes: 89 additions & 0 deletions benchmarks/test_expr.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
import numpy as np

from pyscipopt import Model, cos, exp, log, quicksum, sin, sqrt


def _vars(n):
m = Model()
return m, [m.addVar() for _ in range(n)]


def test_quicksum(bench):
_, x = _vars(100_000)
bench(lambda: quicksum(c * v for c in range(1, 9) for v in x))


def test_expr_arithmetic(bench):
_, x = _vars(8000)

def run():
e = 0
for i, v in enumerate(x):
e = e + 3 * v - i
return e <= 1

bench(run)


def test_repeated_terms(bench):
_, x = _vars(100)
bench(lambda: quicksum(float(i) * x[i % 100] for i in range(1_000_000)))


def test_expr_ops(bench):
_, x = _vars(3000)
exprs = [x[i] + 2 * x[i + 1] - 3 for i in range(len(x) - 1)]

def run():
for _ in range(30):
for e, f in zip(exprs, exprs[1:]):
-e
3.0 * e
e + f
e - f
e * f

bench(run)


def test_quadratic_expr(bench):
_, x = _vars(1000)
bench(lambda: quicksum(x[i] * x[j] for i in range(len(x)) for j in range(i, len(x))))


def test_nonlinear_expr(bench):
_, x = _vars(10_000)
bench(lambda: quicksum(exp(v) + sqrt(v + 1) for v in x) <= 10)


def test_evaluate_nonlinear(bench):
m, x = _vars(3000)
exprs = [exp(v) + log(v + 1) + sqrt(v + 2) + sin(v) + cos(v) + v**2.5 + abs(v - 1) for v in x]
sol = m.createSol()
for i, v in enumerate(x):
m.setSolVal(sol, v, i / len(x))

def run():
for _ in range(100):
for e in exprs:
sol[e]

bench(run)


def test_matrix_ops(bench):
n = 100
m = Model()
x = m.addMatrixVar((n, n))
a = np.arange(n * n, dtype=float).reshape(n, n)

def run():
x.sum()
x.sum(axis=0)
np.add.reduce(x)
-x
2 * x
a @ x
return x <= 1

bench(run)
82 changes: 82 additions & 0 deletions benchmarks/test_model.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
import pytest

from pyscipopt import Model, exp, quicksum, sqrt


def _model_with_vars(n):
m = Model()
m.hideOutput()
return m, [m.addVar(f"x{i}") for i in range(n)]


def test_add_vars(bench):
def run():
m = Model()
for i in range(100_000):
m.addVar(f"x{i}", vtype="I", lb=0, ub=10, obj=1.0)

bench(run)


@pytest.mark.timing
def test_add_linear_conss(bench):
n = 2_000_000

def run(m, x):
for i in range(n - 1):
m.addCons(x[i] + 2 * x[i + 1] <= 10)

bench(run, setup=lambda: (_model_with_vars(n), {}))


def test_add_cons_linear(bench):
n = 300_000

def run(m, x):
for i in range(n - 1):
m.addConsLinear([x[i], x[i + 1]], [1.0, 2.0], rhs=10)

bench(run, setup=lambda: (_model_with_vars(n), {}))


def test_add_nonlinear_conss(bench):
n = 20_000

def run(m, x):
for i in range(n - 1):
m.addCons(exp(x[i]) + x[i] * x[i + 1] + sqrt(x[i + 1] + 1) <= 10)

bench(run, setup=lambda: (_model_with_vars(n), {}))


def test_getters_after_solve(bench):
m, x = _model_with_vars(100_000)
m.setObjective(quicksum(x))
m.optimize()
sol = m.getBestSol()

def run():
for _ in range(6):
for v in x:
m.getVal(v)
m.getSolVal(sol, v)
v.getLbGlobal()
v.getObj()

bench(run)


def test_enum_getters(bench):
m, x = _model_with_vars(1000)
m.setObjective(quicksum(x))
m.optimize()
sol = m.getBestSol()

def run():
for _ in range(1000):
m.getStage()
sol.getOrigin()
for v in x:
v.getImplType()

bench(run)
Loading
Loading