From b269539ca09b2d6096ff79d6b7e8044b900ab422 Mon Sep 17 00:00:00 2001 From: Rinchin Date: Fri, 30 Oct 2026 16:38:29 +0300 Subject: [PATCH 1/9] Add a frozen cross-version local GPU benchmark suite Define twelve deterministic workloads, including output scaling from 32 to 10000, multiple sketch strategies, validation memory and deep histograms. Measure synchronized wall time, private-pool peak live allocations, RSS and holdout quality in isolated workers. Store versioned raw evidence and reject incompatible result comparisons. Document a pinned environment and require local GPU reports in AGENTS.md. --- .gitignore | 1 + AGENTS.md | 33 +++ benchmarks/README.md | 136 +++++++++ benchmarks/__init__.py | 1 + benchmarks/gpu.py | 50 ++++ benchmarks/protocol.py | 134 +++++++++ benchmarks/requirements-py312-cu12.txt | 32 +++ benchmarks/run.py | 372 +++++++++++++++++++++++++ benchmarks/suite-v1.json | 35 +++ tests/test_benchmark_protocol.py | 145 ++++++++++ 10 files changed, 939 insertions(+) create mode 100644 AGENTS.md create mode 100644 benchmarks/README.md create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/gpu.py create mode 100644 benchmarks/protocol.py create mode 100644 benchmarks/requirements-py312-cu12.txt create mode 100644 benchmarks/run.py create mode 100644 benchmarks/suite-v1.json create mode 100644 tests/test_benchmark_protocol.py diff --git a/.gitignore b/.gitignore index 75677ef..0bd61ba 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +.benchmark-results/ cover/ # Translations diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..07018f4 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,33 @@ +# Repository instructions + +уточняй непонятные вопросы перед изменением кода + +## Required local GPU benchmark + +Before declaring a PR ready, run the complete `pyboost-gpu-v1` benchmark described +in `benchmarks/README.md` against the PR's exact base and candidate commits, using +the same harness, pinned environment and idle GPU. Use all 12 cases, one warmup +and five timing repetitions; keep memory runs separate. Do not replace the suite +with a single sketch method, a small-output case, or an improvised timing script. + +- Commit library changes first. Record both code SHAs and the benchmark/data + checksums. Verify that the imported library comes from the intended checkout. +- Run `benchmarks/run.py run` with both source paths, an explicit device and a new + output directory. A missing GPU, failed case, timeout or OOM is not a pass. +- Use `benchmarks/run.py compare` to produce the report; do not compare mismatched + environments/workloads. Inspect time, peak live CuPy bytes, CPU RSS, holdout + quality, executed iterations and retained tree counts. +- When PR creation/update is authorized, publish the comparison table in the PR + and link both raw result sets from a permanent location, preferably the separate + `benchmark-results` branch. Publishing reports must not change the measured + library commit. For local-only work, provide paths to the saved reports. +- After changing the library code, rerun against the new candidate SHA. An earlier + report does not verify later code changes. For a change to the harness itself, + rerun both library versions with the final harness implementation. +- Explicitly report regressions, failures, omitted cases and unavailable hardware. + Preserve completed work and evidence if a run cannot finish; do not describe + such a PR as benchmark-verified. Request an explicit user waiver when needed. + +The frozen benchmark is for version-to-version comparisons. Keep its cases, +generation and measurement semantics stable; introduce a new suite version when +these semantics change. GPU unit tests remain required and do not replace it. diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..76c275f --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,136 @@ +# Py-Boost GPU benchmark v1 + +`pyboost-gpu-v1` is a fixed cross-version benchmark, not a collection of ad hoc +experiments. Run this harness against clean Git checkouts using the same Python +environment and one otherwise idle NVIDIA GPU. A GitHub Actions GPU runner is +not required. The tested library version is selected by `--source`, independently +of the directory containing the harness. + +## Frozen workload + +Definitions are in `suite-v1.json`. All cases build **32 iterations**, use learning +rate 0.1, seed 42, full row/column sampling, and `use_hess=True`. Defaults are depth +4 and 64 bins. Sketch cases keep 8 outputs for structure search. Early stopping is +disabled. The benchmark counts executed iterations and separately records retained +trees: existing Py-Boost versions can prune to the best validation iteration even +with `es=0`. Different retained tree counts are disclosed in comparisons. + +| Case | Train / validation / test rows | Features | Outputs | Model | +|---|---:|---:|---:|---| +| mse-single | 16384 / 4096 / 4096 | 32 | 1 | GradientBoosting | +| mse-full-128 | 8192 / 2048 / 2048 | 32 | 128 | GradientBoosting | +| mse-proj-32 | 16384 / 4096 / 4096 | 32 | 32 | SketchBoost/proj | +| mse-proj-128 | 16384 / 4096 / 4096 | 32 | 128 | SketchBoost/proj | +| mse-proj-1024 | 16384 / 4096 / 4096 | 32 | 1024 | SketchBoost/proj | +| mse-proj-10000 | 16384 / 4096 / 4096 | 32 | 10000 | SketchBoost/proj | +| bce-rand-1024 | 16384 / 4096 / 4096 | 32 | 1024 | SketchBoost/rand | +| bce-filter-1024 | 16384 / 4096 / 4096 | 32 | 1024 | SketchBoost/filter | +| mse-topk-10000 | 4096 / 1024 / 1024 | 32 | 10000 | SketchBoost/topk | +| ce-proj-512 | 16384 / 4096 / 4096 | 32 | 512 classes | SketchBoost/proj | +| validation-proj-1024 | 8192 / 32768 / 4096 | 32 | 1024 | SketchBoost/proj; two validation sets | +| histogram-topk-128 | 8192 / 2048 / 2048 | 128 | 128 | SketchBoost/topk; depth 8, 256 bins | + +The four `mse-proj-*` cases change only the number of outputs. Other cases exercise +full-output scoring, separate/shared Hessians, wide targets, validation memory, +and deep histograms. No sketch method is treated as the primary benchmark. +SVD requires an additional RAPIDS stack and is outside v1. Missing-label losses +are outside v1 to keep this suite runnable against released 0.5.2 as well as newer +versions; add such workloads under a new suite ID. + +Data are generated by `protocol.make_data` using PCG64 seed 20261030, float32 +features/targets and eight correlated output groups. Regression has additive +noise; multilabel targets are Bernoulli draws from signal-dependent probabilities. +Multiclass labels encode feature signs, with all classes represented in training. +Generation uses no matrix multiplication or external downloads. SHA-256 covers +the actual feature and target bytes. Validation/test rows are held out. The rand +and filter cases receive identical data. + +## Environment and commands + +The reference stack is Linux, Python 3.12 and CUDA 12. Install the complete pinned +dependency closure in a dedicated environment. This file deliberately does not +install a particular Py-Boost revision; install metadata once, then select each +tested checkout through `--source`. + +```bash +uv venv --python 3.12 .venv-benchmark +uv pip install --python .venv-benchmark/bin/python -r benchmarks/requirements-py312-cu12.txt +uv pip install --python .venv-benchmark/bin/python --no-deps -e . +git worktree add --detach ../Py-Boost-v0.5.2 v0.5.2 +.venv-benchmark/bin/python benchmarks/run.py run \ + --source ../Py-Boost-v0.5.2 --source . \ + --device 1 --output .benchmark-results/v052-vs-current +``` + +Commit changes to `py_boost/` or `pyproject.toml` first: dirty library sources are +rejected. Use a physical GPU index or UUID (`nvidia-smi -L`). Do not share that GPU +with another workload. Preflight performs a real allocation/kernel, rejects other +compute processes on the GPU, and records its UUID. CPU thread counts are fixed at +4. No GPU is a failure, not a skipped or CPU-emulated benchmark. + +Each source produces `//results.json` and raw worker +requests/results/logs. Two-source runs also produce `comparison.md`. More than two +sources may be measured in one run; compare any two results afterward: + +```bash +.venv-benchmark/bin/python benchmarks/run.py compare \ + .benchmark-results/older/COMMIT/results.json \ + .benchmark-results/newer/COMMIT/results.json \ + --output .benchmark-results/comparison.md +``` + +`--python /path/to/environment/bin/python` selects the worker interpreter. +`--case mse-single --repeats 1` is a diagnostic subset, not the mandatory full +profile. Output directories must be empty so previous evidence is not overwritten. +Workers have a 900-second timeout including setup/warmup; failures and OOM retain +logs/results, and the command exits nonzero. A version lacking a required API is +reported as failed, not silently replaced by another implementation. + +## Measurement contract + +- Each case/measurement phase/version uses a fresh subprocess. When multiple + versions are measured, their order alternates by case. +- Each subprocess generates and hashes data, then warms one complete fit/predict. + Data generation, imports, JIT warmup and estimator construction are excluded + from timing. Quantization, CPU/GPU transfers and evaluation inside `fit()` are + included. Prediction receives CPU input and includes transfers back to CPU. +- Timing uses five fresh-model repetitions, `perf_counter` and device-wide CUDA + synchronization. Raw samples, median and p10/p90 are retained. These are warmed + timings, not cold-start latency. Holdout scoring is outside the timers. +- Memory uses a separate run so hooks/RSS sampling do not contaminate timings. + CuPy malloc/free hooks track **peak live rounded allocation bytes**, including + Py-Boost's private fit pool. This excludes free pool cache, CUDA context and + allocations outside CuPy's hooked pools; it is not total process VRAM. + Prediction tracking includes still-live allocations observed during fit. +- CPU RSS is sampled every 5 ms during fit/predict and includes the resident input + dataset. It is an approximation and may miss very short peaks. Absolute peaks + and the fit baseline are stored. +- Holdout RMSE or log loss is computed in bounded float64 blocks. Every prediction + must be finite and have the expected shape. Metric samples, actual iteration + counts and retained tree counts are stored, so faster runs can be assessed + together with model quality and work performed. + +## Cross-version comparability and publication + +Reports identify the exact library commit and library-source checksum, benchmark +implementation checksum, manifest checksum, actual dataset checksum, GPU UUID, +CPU/OS, Python, relevant dependency versions, CUDA driver/runtime, repetitions and +warmups. `compare` refuses mismatched workloads, datasets, environments or failed +measurements (exit code 2). Partial runs are marked `full_suite: false`; only a full +five-repeat run satisfies the PR requirement. Comparison tables show measured +changes, not an automatic statistical pass/fail verdict. + +To compare historical versions, run them through **this same harness** and pinned +environment. Different GPUs/environments need separate baselines. Repeated runs +can still differ due to temperature, clocks and scheduling: retain raw samples +and investigate noise before calling a small difference a regression. + +Treat v1's data, cases and measurement semantics as frozen. New semantics/workloads +need a new suite version. Any harness change alters its checksum and requires +remeasuring both versions; never relabel old results to bypass that check. + +Publish `comparison.md` in the PR and retain both `results.json` files and raw logs +in the repository's separate `benchmark-results` branch or another explicitly +agreed permanent location. Keep the measured code SHA unchanged when publishing +reports. The runner itself performs no network writes. Do not commit local results +or environment directories into the code branch. diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..2252207 --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1 @@ +"""Versioned, standalone benchmarks; not part of the py_boost package.""" diff --git a/benchmarks/gpu.py b/benchmarks/gpu.py new file mode 100644 index 0000000..5ba8d59 --- /dev/null +++ b/benchmarks/gpu.py @@ -0,0 +1,50 @@ +"""Measurement helpers. GPU timing and allocation tracking run separately.""" + +import threading + + +def memory_tracker(cp): + class Tracker(cp.cuda.MemoryHook): + def __init__(self): + self.live = {} + self.current = 0 + self.peak = 0 + + def malloc_postprocess(self, device_id, size, mem_size, mem_ptr, pmem_id): + if mem_ptr and pmem_id: + key = (device_id, pmem_id) + self.current += mem_size - self.live.get(key, 0) + self.live[key] = mem_size + self.peak = max(self.peak, self.current) + + def free_postprocess(self, device_id, mem_size, mem_ptr, pmem_id): + self.current -= self.live.pop((device_id, pmem_id), 0) + + def reset_peak(self): + self.peak = self.current + + return Tracker() + + +class RssPeak: + """Sample absolute RSS during one phase; includes resident CPU input data.""" + + def __init__(self, process): + self.process = process + self.stop = threading.Event() + + def __enter__(self): + self.baseline = self.process.memory_info().rss + self.peak = self.baseline + self.thread = threading.Thread(target=self._sample, daemon=True) + self.thread.start() + return self + + def _sample(self): + while not self.stop.wait(.005): + self.peak = max(self.peak, self.process.memory_info().rss) + + def __exit__(self, *exc): + self.peak = max(self.peak, self.process.memory_info().rss) + self.stop.set() + self.thread.join() diff --git a/benchmarks/protocol.py b/benchmarks/protocol.py new file mode 100644 index 0000000..eff4d27 --- /dev/null +++ b/benchmarks/protocol.py @@ -0,0 +1,134 @@ +"""Data and result contracts shared by every tested library version.""" + +import hashlib +import json +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parent +SCHEMA_VERSION = 1 + + +def digest_json(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest() + + +def implementation_hash(): + digest = hashlib.sha256() + for name in ['suite-v1.json', 'protocol.py', 'gpu.py', 'run.py']: + digest.update(name.encode()) + digest.update((ROOT / name).read_bytes()) + return digest.hexdigest() + + +def load_suite(): + return json.loads((ROOT / 'suite-v1.json').read_text()) + + +def make_data(case, seed): + """PCG64, float32 inputs, eight correlated output groups, no external data.""" + rng = np.random.Generator(np.random.PCG64(seed)) + n = case['train'] + case['valid'] + case['test'] + X = rng.standard_normal((n, case['features']), dtype=np.float32) + outputs = case['outputs'] + if case['task'] == 'multiclass': + bits = (outputs - 1).bit_length() + y = np.zeros(n, dtype=np.int32) + for bit in range(bits): + y |= (X[:, bit] > 0).astype(np.int32) << bit + y %= outputs + # Ensure every class is represented in training, with consistent features. + for bit in range(bits): + X[:outputs, bit] = np.where((np.arange(outputs) >> bit) & 1, .5, -.5) + y[:outputs] = np.arange(outputs) + else: + latent = np.stack([X[:, 0], X[:, 1], X[:, 2] * X[:, 3], + np.maximum(X[:, 4], 0), np.sin(X[:, 5]), + X[:, 6] + X[:, 7], X[:, 8] - X[:, 9], + (X[:, 10] > 0).astype(np.float32) - .5], axis=1) + y = np.empty((n, outputs), dtype=np.float32) + for start in range(0, outputs, 128): + stop = min(start + 128, outputs) + columns = np.arange(start, stop) + signal = latent[:, columns % 8] + .3 * latent[:, (columns + 3) % 8] + noise = rng.standard_normal(signal.shape, dtype=np.float32) + if case['task'] == 'regression': + y[:, start:stop] = signal + .1 * noise + else: + probability = 1 / (1 + np.exp(-signal)) + y[:, start:stop] = rng.random(signal.shape, dtype=np.float32) < probability + digest = hashlib.sha256() + for arr in [X, y]: + digest.update(str((arr.shape, arr.dtype.str)).encode()) + digest.update(memoryview(arr).cast('B')) + t, v = case['train'], case['train'] + case['valid'] + validation = [] + for indices in np.array_split(np.arange(t, v), case['validation_sets']): + validation.append({'X': X[indices[0]:indices[-1] + 1], + 'y': y[indices[0]:indices[-1] + 1]}) + return X[:t], y[:t], validation, X[v:], y[v:], digest.hexdigest() + + +def quality(task, target, prediction): + """Holdout metric outside timed regions, with bounded float64 temporaries.""" + if not np.isfinite(prediction).all(): + raise ValueError('Non-finite predictions') + if prediction.shape[0] != target.shape[0]: + raise ValueError('Prediction row count differs from target') + total, count = 0., 0 + for start in range(0, len(target), 256): + y = target[start:start + 256] + p = prediction[start:start + 256].astype(np.float64) + if task == 'multiclass': + if p.ndim != 2 or (y.max() >= p.shape[1]): + raise ValueError('Missing class probabilities') + values = -np.log(np.clip(p[np.arange(len(y)), y], 1e-15, 1)) + elif task == 'regression': + if p.shape != y.shape: + raise ValueError('Prediction output count differs from target') + values = (p - y.astype(np.float64)) ** 2 + else: + if p.shape != y.shape: + raise ValueError('Prediction output count differs from target') + p = np.clip(p, 1e-15, 1 - 1e-15) + values = -(y * np.log(p) + (1 - y) * np.log1p(-p)) + total += float(values.sum()) + count += values.size + value = total / count + return {'name': 'rmse' if task == 'regression' else 'logloss', + 'value': float(np.sqrt(value) if task == 'regression' else value), + 'lower_is_better': True} + + +def summarize(values): + return {'median': float(np.median(values)), 'p10': float(np.quantile(values, .1)), + 'p90': float(np.quantile(values, .9)), 'samples': values} + + +def comparison_errors(base, head): + errors = [] + for key in ['schema_version', 'benchmark_id', 'implementation_hash', 'suite_hash', + 'case_ids', 'repeats', 'warmups']: + if base.get(key) != head.get(key): + errors.append(f'Different {key}') + if base.get('environment') != head.get('environment'): + errors.append('Different hardware or software environment') + for label, report in [('base', base), ('candidate', head)]: + if not report.get('case_ids') or set(report.get('case_ids', [])) != set(report.get('cases', {})): + errors.append(f'{label}: missing case results') + for case_id in sorted(set(base.get('cases', {})) & set(head.get('cases', {}))): + a, b = base['cases'][case_id], head['cases'][case_id] + if a.get('status') != 'ok' or b.get('status') != 'ok': + errors.append(f'{case_id}: failed or incomplete measurement') + elif a.get('data_hash') != b.get('data_hash'): + errors.append(f'{case_id}: different dataset checksum') + elif not a.get('data_hash'): + errors.append(f'{case_id}: missing dataset checksum') + if a.get('spec') != b.get('spec'): + errors.append(f'{case_id}: different case parameters') + if any(c.get('status') != 'ok' for c in list(base.get('cases', {}).values()) + list(head.get('cases', {}).values())): + errors.append('One or more cases did not complete') + if not base.get('complete') or not head.get('complete'): + errors.append('A report is incomplete') + return errors diff --git a/benchmarks/requirements-py312-cu12.txt b/benchmarks/requirements-py312-cu12.txt new file mode 100644 index 0000000..6e4cca6 --- /dev/null +++ b/benchmarks/requirements-py312-cu12.txt @@ -0,0 +1,32 @@ +# Tested Linux/Python 3.12/CUDA 12 dependency closure; library checkout is selected separately. +cloudpickle==3.1.2 +cuda-pathfinder==1.8.5 +cupy-cuda12x==14.2.0 +flatbuffers==25.12.19 +iniconfig==2.3.1 +joblib==1.6.0 +llvmlite==0.50.0 +ml-dtypes==0.6.0 +narwhals==2.28.0 +numba==0.68.1 +numpy==2.5.4 +onnx==1.23.2 +onnxruntime==1.31.1 +packaging==26.3 +pandas==3.1.0 +pluggy==1.7.0 +protobuf==7.36.2 +psutil==7.2.2 +pygments==2.21.0 +pytest==9.2.0 +python-dateutil==3.0.0 +scikit-learn==1.9.1 +scipy==1.18.1 +six==1.17.0 +threadpoolctl==3.7.0 +tqdm==4.70.1 +treelite==3.9.1 +treelite-runtime==3.9.1 +typing-extensions==4.16.0 +tzdata==2026.6 +ujson==6.0.0 diff --git a/benchmarks/run.py b/benchmarks/run.py new file mode 100644 index 0000000..4fd9b7c --- /dev/null +++ b/benchmarks/run.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +"""Run a frozen GPU suite against one or more Py-Boost source checkouts.""" + +import argparse +import datetime +import gc +import hashlib +import importlib.metadata +import json +import logging +import os +from pathlib import Path +import platform +import subprocess +import sys +import time +import traceback + +if __package__: + from .protocol import (ROOT, SCHEMA_VERSION, comparison_errors, digest_json, + implementation_hash, load_suite, make_data, quality, summarize) + from .gpu import RssPeak, memory_tracker +else: + from protocol import (ROOT, SCHEMA_VERSION, comparison_errors, digest_json, + implementation_hash, load_suite, make_data, quality, summarize) + from gpu import RssPeak, memory_tracker + + +def write_json(path, data): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + '.tmp') + temporary.write_text(json.dumps(data, indent=2, sort_keys=True, allow_nan=False) + '\n') + temporary.replace(path) + + +def git(source, *args): + return subprocess.check_output(['git', '-C', str(source), *args], text=True).strip() + + +def source_info(source): + source = Path(source).resolve() + if not (source / 'py_boost' / '__init__.py').is_file(): + raise ValueError(f'Not a Py-Boost checkout: {source}') + paths = ['py_boost', 'pyproject.toml'] + if git(source, 'status', '--porcelain', '--untracked-files=all', '--', *paths): + raise ValueError(f'Commit library changes before benchmarking: {source}') + digest = hashlib.sha256() + for name in git(source, 'ls-files', '--', *paths).splitlines(): + digest.update(name.encode()) + digest.update((source / name).read_bytes()) + return {'path': str(source), 'commit': git(source, 'rev-parse', 'HEAD'), + 'library_hash': digest.hexdigest()} + + +def environment(cp, np, psutil, device): + if cp.cuda.runtime.getDeviceCount() != 1: + raise RuntimeError('Exactly one visible CUDA device is required') + cp.zeros(1).sum().item() # A real allocation and kernel; no CPU fallback. + props = cp.cuda.runtime.getDeviceProperties(0) + name = props['name'] + uuid = subprocess.check_output( + ['nvidia-smi', '-i', device, '--query-gpu=uuid', '--format=csv,noheader'], text=True).strip() + processes = subprocess.check_output( + ['nvidia-smi', '--query-compute-apps=gpu_uuid,pid', '--format=csv,noheader,nounits'], + text=True) + busy = [line for line in processes.splitlines() + if line.split(',')[0].strip() == uuid and line.split(',')[1].strip() != str(os.getpid())] + if busy: + raise RuntimeError(f'GPU is in use by other compute processes: {busy}') + packages = {} + for distribution in importlib.metadata.distributions(): + package = distribution.metadata['Name'] + if package and package.lower().replace('_', '-') in { + 'numpy', 'scipy', 'numba', 'llvmlite', 'scikit-learn', 'joblib', 'pandas', + 'cupy', 'cupy-cuda11x', 'cupy-cuda12x', 'cupy-cuda13x', 'psutil', + 'treelite', 'treelite-runtime', 'onnx', 'onnxruntime', 'ujson', + }: + packages[package.lower().replace('_', '-')] = distribution.version + cpu = platform.processor() + if Path('/proc/cpuinfo').exists(): + cpu = next((line.split(':', 1)[1].strip() for line in + Path('/proc/cpuinfo').read_text().splitlines() if line.startswith('model name')), cpu) + return {'python': platform.python_version(), 'platform': platform.platform(), + 'cpu': cpu, 'cpu_threads': int(os.environ['NUMBA_NUM_THREADS']), + 'host_ram_bytes': psutil.virtual_memory().total, + 'gpu': {'name': name.decode() if isinstance(name, bytes) else name, + 'uuid': uuid, 'vram_bytes': props['totalGlobalMem'], + 'compute_capability': [props['major'], props['minor']]}, + 'cuda_driver': cp.cuda.runtime.driverGetVersion(), + 'cuda_runtime': cp.cuda.runtime.runtimeGetVersion(), 'packages': packages} + + +def worker(job): + import numpy as np + import cupy as cp + import psutil + + logging.disable(logging.CRITICAL) + cp.cuda.Device(0).use() + metadata = environment(cp, np, psutil, job['device']) + if job['phase'] == 'metadata': + return {'status': 'ok', 'environment': metadata} + sys.path.insert(0, job['source']['path']) + import py_boost + expected = Path(job['source']['path']) + if Path(py_boost.__file__).resolve().parents[1] != expected: + raise RuntimeError(f'Imported the wrong library: {py_boost.__file__}') + if source_info(expected) != job['source']: + raise RuntimeError('Library source changed after the run started') + case, suite = job['case'], job['suite'] + X, y, valid, test_X, test_y, data_hash = make_data(case, suite['data_seed']) + params = dict(suite['common_params'], **case['params']) + params.update(loss=case['loss'], seed=suite['model_seed']) + if case['model'] == 'SketchBoost': + params['sketch_params'] = suite['sketch_params'][params['sketch_method']] + else: + params['target_splitter'] = 'Single' + + class Iterations(py_boost.Callback): + count = 0 + + def after_iteration(self, info): + self.count += 1 + return False + + def new_model(): + counter = Iterations() + model = getattr(py_boost, case['model'])(**params, callbacks=[counter]) + return model, counter + + def cleanup(model): + model.to_cpu() + gc.collect() + cp.get_default_memory_pool().free_all_blocks() + cp.get_default_pinned_memory_pool().free_all_blocks() + + def sync(): + cp.cuda.Device(0).synchronize() + + for _ in range(suite['warmups']): + warm, _ = new_model() + warm.fit(X, y, eval_sets=valid) + warm.predict(test_X) + sync() + cleanup(warm) + del warm + gc.collect() + cp.get_default_memory_pool().free_all_blocks() + fit_times, predict_times, measurements = [], [], [] + repeats = job['repeats'] if job['phase'] == 'timing' else 1 + for _ in range(repeats): + model, counter = new_model() + if job['phase'] == 'timing': + sync() + start = time.perf_counter() + model.fit(X, y, eval_sets=valid) + sync() + fit_times.append(time.perf_counter() - start) + start = time.perf_counter() + prediction = model.predict(test_X) + sync() + predict_times.append(time.perf_counter() - start) + else: + with memory_tracker(cp) as tracker: + with RssPeak(psutil.Process()) as rss_fit: + model.fit(X, y, eval_sets=valid) + sync() + fit_peak = tracker.peak + tracker.reset_peak() + with RssPeak(psutil.Process()) as rss_predict: + prediction = model.predict(test_X) + sync() + memory = {'fit_cupy_peak_bytes': fit_peak, + 'predict_cupy_peak_bytes': tracker.peak, + 'fit_rss_peak_bytes': rss_fit.peak, + 'predict_rss_peak_bytes': rss_predict.peak, + 'fit_rss_baseline_bytes': rss_fit.baseline} + if counter.count != params['ntrees']: + raise RuntimeError(f'Expected {params["ntrees"]} iterations, got {counter.count}') + if prediction.shape != (case['test'], case['outputs']): + raise RuntimeError(f'Unexpected prediction shape: {prediction.shape}') + score = quality(case['task'], test_y, prediction) + if not np.isfinite(score['value']): + raise RuntimeError('Non-finite holdout metric') + measurements.append({'trained_iterations': counter.count, + 'retained_trees': len(model.models), 'quality': score}) + cleanup(model) + del model, prediction + result = {'status': 'ok', 'environment': metadata, 'data_hash': data_hash, + 'params': params, 'measurements': measurements, + 'quality': dict(name=score['name'], **summarize([m['quality']['value'] for m in measurements])), + 'imported_library': py_boost.__file__} + if job['phase'] == 'timing': + result.update(fit_seconds=summarize(fit_times), predict_seconds=summarize(predict_times)) + else: + result.update(memory) + return result + + +def invoke(python, output, job, timeout): + output = Path(output) + request, result, log = output.with_suffix('.job.json'), output.with_suffix('.json'), output.with_suffix('.log') + write_json(request, job) + env = dict(os.environ) + env['CUDA_VISIBLE_DEVICES'] = job['device'] + for key in ['OMP_NUM_THREADS', 'OPENBLAS_NUM_THREADS', 'MKL_NUM_THREADS', 'NUMBA_NUM_THREADS']: + env[key] = str(job['suite']['cpu_threads']) + env['PYTHONHASHSEED'] = '0' + env['PYTHONPATH'] = job['source']['path'] + try: + with log.open('w') as stream: + process = subprocess.run( + [python, str(Path(__file__).resolve()), '_worker', str(request), str(result)], + stdout=stream, stderr=subprocess.STDOUT, env=env, timeout=timeout, + cwd=job['source']['path']) + if result.exists(): + record = json.loads(result.read_text()) + else: + record = {'status': 'error', 'error': f'Worker exited {process.returncode} without a result'} + if process.returncode and record.get('status') == 'ok': + record = {'status': 'error', 'error': f'Worker exited {process.returncode}'} + except subprocess.TimeoutExpired: + record = {'status': 'error', 'error': f'Timeout after {timeout} seconds'} + record['log'] = str(log) + return record + + +def run(args): + suite = load_suite() + sources = [source_info(p) for p in args.source] + if len({s['commit'] for s in sources}) != len(sources): + raise ValueError('Each source must have a different commit') + selected = args.case or [c['id'] for c in suite['cases']] + if set(selected) - {c['id'] for c in suite['cases']}: + raise ValueError('Unknown case ID') + if args.repeats < 1 or args.timeout < 1: + raise ValueError('Repeats and timeout must be positive') + if ',' in args.device or not args.device: + raise ValueError('Select exactly one GPU index or UUID') + cases = [c for c in suite['cases'] if c['id'] in selected] + output = Path(args.output).resolve() + output.mkdir(parents=True, exist_ok=True) + if any(output.iterdir()): + raise ValueError('Output directory must be empty; preserve previous measurements') + reports, directories = [], [] + for source in sources: + directory = output / source['commit'][:12] + (directory / 'raw').mkdir(parents=True) + report = {'schema_version': SCHEMA_VERSION, 'benchmark_id': suite['benchmark_id'], + 'implementation_hash': implementation_hash(), 'suite_hash': digest_json(suite), + 'started_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'source': source, 'case_ids': [c['id'] for c in cases], + 'full_suite': len(cases) == len(suite['cases']), + 'repeats': args.repeats, 'warmups': suite['warmups'], + 'suite': suite, 'cases': {}, 'complete': False} + job = dict(source=source, phase='metadata', device=args.device, suite=suite) + probe = invoke(args.python, directory / 'raw' / 'metadata', job, args.timeout) + if probe['status'] != 'ok': + report['error'] = probe + write_json(directory / 'results.json', report) + raise RuntimeError(f'GPU/environment preflight failed; see {probe["log"]}') + report['environment'] = probe['environment'] + reports.append(report) + directories.append(directory) + write_json(directory / 'results.json', report) + for index, case in enumerate(cases): + # Alternate version order for each case; each case/phase gets a fresh process. + order = list(range(len(sources))) + if index % 2: + order.reverse() + for phase in ['timing', 'memory']: + for i in order: + print(f'{sources[i]["commit"][:12]} {case["id"]} {phase}', flush=True) + job = dict(source=sources[i], phase=phase, device=args.device, suite=suite, + case=case, repeats=args.repeats) + record = invoke(args.python, directories[i] / 'raw' / f'{case["id"]}-{phase}', job, args.timeout) + entry = reports[i]['cases'].setdefault(case['id'], {'spec': case, 'status': 'running'}) + entry[phase] = record + if record['status'] == 'ok' and record['environment'] != reports[i]['environment']: + record.update(status='error', error='Environment changed during the run') + if phase == 'memory': + a, b = entry['timing'], entry['memory'] + entry['status'] = 'ok' if a['status'] == b['status'] == 'ok' else 'error' + if entry['status'] == 'ok': + entry['data_hash'] = a['data_hash'] + if a['data_hash'] != b['data_hash']: + entry.update(status='error', error='Dataset changed between measurement phases') + write_json(directories[i] / 'results.json', reports[i]) + for report, directory in zip(reports, directories): + report['complete'] = all(c['status'] == 'ok' for c in report['cases'].values()) + report['finished_utc'] = datetime.datetime.now(datetime.timezone.utc).isoformat() + write_json(directory / 'results.json', report) + if len(reports) == 2: + (output / 'comparison.md').write_text(render_comparison(*reports)) + return 0 if all(r['complete'] for r in reports) else 1 + + +def render_comparison(base, head): + errors = comparison_errors(base, head) + lines = [f'# {base.get("benchmark_id", "GPU benchmark")}', '', + f'Base: `{base["source"]["commit"]}`; candidate: `{head["source"]["commit"]}`.', ''] + if errors: + return '\n'.join(lines + ['**Not comparable:**'] + [f'- {e}' for e in errors]) + '\n' + lines += ['Times are warmed wall-clock medians. GPU memory is peak live CuPy allocations,', + 'including the private fit pool; CUDA context, pool cache and external library allocations are excluded.', + 'Quality is a holdout metric (lower is better). These are measured changes, not statistical verdicts.', '', + '| Case | Fit s (base → PR) | Δ fit | Predict s (base → PR) | Δ predict | Fit MiB (base → PR) | Δ memory | Quality (base → PR) |', + '|---|---:|---:|---:|---:|---:|---:|---:|'] + notes = [] + for case_id in base['case_ids']: + a, b = base['cases'][case_id], head['cases'][case_id] + def pair(section, key, scale=1): + x, y = a[section][key], b[section][key] + if isinstance(x, dict): + x, y = x['median'], y['median'] + delta = f'{(y / x - 1) * 100:+.1f}%' if x else 'n/a' + return f'{x / scale:.4f} → {y / scale:.4f}', delta + fit, df = pair('timing', 'fit_seconds') + pred, dp = pair('timing', 'predict_seconds') + mem, dm = pair('memory', 'fit_cupy_peak_bytes', 2**20) + qa = a['timing']['quality']['median'] + qb = b['timing']['quality']['median'] + lines.append(f'| {case_id} | {fit} | {df} | {pred} | {dp} | {mem} | {dm} | {qa:.6g} → {qb:.6g} |') + ra = [m['retained_trees'] for m in a['timing']['measurements']] + rb = [m['retained_trees'] for m in b['timing']['measurements']] + if len(set(ra + rb)) > 1: + notes.append(f'Retained tree counts differ for `{case_id}`: {ra} vs {rb}; interpret prediction timing with care.') + return '\n'.join(lines + [''] + notes) + '\n' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + command = commands.add_parser('run', help='Run one or more clean source checkouts') + command.add_argument('--source', action='append', required=True, help='Git checkout; repeat to compare versions') + command.add_argument('--output', required=True) + command.add_argument('--device', required=True, help='One physical GPU index or UUID') + command.add_argument('--python', default=sys.executable, help='Interpreter containing the pinned dependencies') + command.add_argument('--repeats', type=int, default=load_suite()['repeats']) + command.add_argument('--case', action='append', help='Diagnostic subset; omit for the full suite') + command.add_argument('--timeout', type=int, default=900, help='Seconds per worker, including warmup') + command = commands.add_parser('compare', help='Compare two archived results.json files') + command.add_argument('base') + command.add_argument('candidate') + command.add_argument('--output', required=True) + command = commands.add_parser('_worker', help=argparse.SUPPRESS) + command.add_argument('job') + command.add_argument('result') + args = parser.parse_args() + if args.command == '_worker': + try: + result = worker(json.loads(Path(args.job).read_text())) + write_json(args.result, result) + return 0 + except Exception: + write_json(args.result, {'status': 'error', 'error': traceback.format_exc()}) + traceback.print_exc() + return 1 + if args.command == 'compare': + base, head = [json.loads(Path(p).read_text()) for p in [args.base, args.candidate]] + Path(args.output).write_text(render_comparison(base, head)) + return 2 if comparison_errors(base, head) else 0 + return run(args) + + +if __name__ == '__main__': + try: + sys.exit(main()) + except (ValueError, RuntimeError, subprocess.SubprocessError) as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) diff --git a/benchmarks/suite-v1.json b/benchmarks/suite-v1.json new file mode 100644 index 0000000..ac87722 --- /dev/null +++ b/benchmarks/suite-v1.json @@ -0,0 +1,35 @@ +{ + "benchmark_id": "pyboost-gpu-v1", + "data_seed": 20261030, + "model_seed": 42, + "cpu_threads": 4, + "warmups": 1, + "repeats": 5, + "sketch_params": { + "proj": {"norm": true}, + "rand": {"smooth": 0.1, "replace": true}, + "filter": {"sample": true, "smooth": 0.1, "ntrees": 1}, + "topk": {} + }, + "common_params": { + "ntrees": 32, "max_depth": 4, "max_bin": 64, "lr": 0.1, + "es": 0, "verbose": 1000000, "min_data_in_leaf": 10, + "colsample": 1.0, "subsample": 1.0, "use_hess": true, + "lambda_l2": 1.0, "gd_steps": 1, "min_gain_to_split": 0, + "quantization": "Quantile", "quant_sample": 2000000, "min_data_in_bin": 3 + }, + "cases": [ + {"id": "mse-single", "task": "regression", "model": "GradientBoosting", "loss": "mse", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 1, "validation_sets": 1, "params": {}}, + {"id": "mse-full-128", "task": "regression", "model": "GradientBoosting", "loss": "mse", "train": 8192, "valid": 2048, "test": 2048, "features": 32, "outputs": 128, "validation_sets": 1, "params": {}}, + {"id": "mse-proj-32", "task": "regression", "model": "SketchBoost", "loss": "mse", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 32, "validation_sets": 1, "params": {"sketch_method": "proj", "sketch_outputs": 8}}, + {"id": "mse-proj-128", "task": "regression", "model": "SketchBoost", "loss": "mse", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 128, "validation_sets": 1, "params": {"sketch_method": "proj", "sketch_outputs": 8}}, + {"id": "mse-proj-1024", "task": "regression", "model": "SketchBoost", "loss": "mse", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 1024, "validation_sets": 1, "params": {"sketch_method": "proj", "sketch_outputs": 8}}, + {"id": "mse-proj-10000", "task": "regression", "model": "SketchBoost", "loss": "mse", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 10000, "validation_sets": 1, "params": {"sketch_method": "proj", "sketch_outputs": 8}}, + {"id": "bce-rand-1024", "task": "multilabel", "model": "SketchBoost", "loss": "multilabel", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 1024, "validation_sets": 1, "params": {"sketch_method": "rand", "sketch_outputs": 8}}, + {"id": "bce-filter-1024", "task": "multilabel", "model": "SketchBoost", "loss": "multilabel", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 1024, "validation_sets": 1, "params": {"sketch_method": "filter", "sketch_outputs": 8}}, + {"id": "mse-topk-10000", "task": "regression", "model": "SketchBoost", "loss": "mse", "train": 4096, "valid": 1024, "test": 1024, "features": 32, "outputs": 10000, "validation_sets": 1, "params": {"sketch_method": "topk", "sketch_outputs": 8}}, + {"id": "ce-proj-512", "task": "multiclass", "model": "SketchBoost", "loss": "crossentropy", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 512, "validation_sets": 1, "params": {"sketch_method": "proj", "sketch_outputs": 8}}, + {"id": "validation-proj-1024", "task": "regression", "model": "SketchBoost", "loss": "mse", "train": 8192, "valid": 32768, "test": 4096, "features": 32, "outputs": 1024, "validation_sets": 2, "params": {"sketch_method": "proj", "sketch_outputs": 8}}, + {"id": "histogram-topk-128", "task": "regression", "model": "SketchBoost", "loss": "mse", "train": 8192, "valid": 2048, "test": 2048, "features": 128, "outputs": 128, "validation_sets": 1, "params": {"sketch_method": "topk", "sketch_outputs": 8, "max_depth": 8, "max_bin": 256}} + ] +} diff --git a/tests/test_benchmark_protocol.py b/tests/test_benchmark_protocol.py new file mode 100644 index 0000000..2cf0b0c --- /dev/null +++ b/tests/test_benchmark_protocol.py @@ -0,0 +1,145 @@ +"""The result protocol must reject misleading cross-version comparisons.""" + +from types import SimpleNamespace +import importlib.util +from pathlib import Path +import sys + +import numpy as np +import pytest + +# The CPU wheel workflow runs tests outside the source root. Load only the +# benchmark package, without putting the library checkout ahead of the wheel. +_spec = importlib.util.spec_from_file_location( + 'pyboost_benchmark_suite', Path(__file__).resolve().parents[1] / 'benchmarks' / '__init__.py') +_package = importlib.util.module_from_spec(_spec) +sys.modules[_spec.name] = _package +_spec.loader.exec_module(_package) + +from pyboost_benchmark_suite.gpu import memory_tracker +from pyboost_benchmark_suite.protocol import comparison_errors, load_suite, make_data, quality +from pyboost_benchmark_suite.run import render_comparison + + +def small_case(task='regression'): + return dict(id='test', train=64, valid=16, test=16, features=32, + outputs=8, validation_sets=2, task=task) + + +@pytest.mark.parametrize('task', ['regression', 'multilabel', 'multiclass']) +def test_datasets_are_reproducible_and_separated(task): + case = small_case(task) + a, b = make_data(case, 42), make_data(case, 42) + assert a[-1] == b[-1] + assert a[-1] != make_data(case, 43)[-1] + np.testing.assert_array_equal(a[0], b[0]) + np.testing.assert_array_equal(a[1], b[1]) + assert a[0].shape == (64, 32) + assert [len(v['y']) for v in a[2]] == [8, 8] + assert not np.shares_memory(a[0], a[3]) + if task == 'multiclass': + np.testing.assert_array_equal(np.unique(a[1]), np.arange(8)) + elif task == 'multilabel': + assert set(np.unique(a[1])) == {0, 1} + else: + assert quality(task, a[1], a[1])['value'] == 0 + + +def test_same_multilabel_data_for_different_sketch_methods(): + suite = load_suite() + rand, filt = [next(c for c in suite['cases'] if c['id'] == name) + for name in ['bce-rand-1024', 'bce-filter-1024']] + for case in [rand, filt]: + case.update(train=64, valid=16, test=16, outputs=8) + assert make_data(rand, suite['data_seed'])[-1] == make_data(filt, suite['data_seed'])[-1] + + +def test_holdout_metrics_and_invalid_predictions(): + assert quality('multilabel', np.array([[0, 1]], dtype=np.float32), + np.array([[.5, .5]], dtype=np.float32))['value'] == pytest.approx(np.log(2)) + assert quality('multiclass', np.array([1]), np.array([[.25, .75]]))['value'] == pytest.approx(-np.log(.75)) + with pytest.raises(ValueError, match='Non-finite'): + quality('regression', np.zeros((1, 1)), np.array([[np.nan]])) + with pytest.raises(ValueError, match='output count'): + quality('regression', np.zeros((1, 2)), np.zeros((1, 1))) + + +def report(): + timing = dict(fit_seconds={'median': 1.}, predict_seconds={'median': .1}, + quality={'median': .2}, measurements=[{'retained_trees': 32}]) + return dict(schema_version=1, benchmark_id='pyboost-gpu-v1', implementation_hash='abc', + suite_hash='def', case_ids=['test'], repeats=5, warmups=1, complete=True, + environment={'gpu': 'A100', 'packages': {'numpy': '2'}}, + source={'commit': 'abc'}, cases={'test': dict(status='ok', data_hash='123', + spec={'outputs': 1024}, timing=timing, memory={'fit_cupy_peak_bytes': 2**20})}) + + +@pytest.mark.parametrize('key,value', [ + ('implementation_hash', 'changed'), ('suite_hash', 'changed'), + ('environment', {'gpu': 'T4'}), ('repeats', 1), ('complete', False), +]) +def test_incompatible_measurements_are_not_compared(key, value): + base, candidate = report(), report() + candidate[key] = value + assert comparison_errors(base, candidate) + text = render_comparison(base, candidate) + assert 'Not comparable' in text + assert '| Case |' not in text + + +@pytest.mark.parametrize('change', ['dataset', 'error', 'missing_case', 'spec']) +def test_incomplete_or_different_workloads_are_rejected(change): + base, candidate = report(), report() + if change == 'dataset': + candidate['cases']['test']['data_hash'] = 'different' + elif change == 'error': + candidate['cases']['test']['status'] = 'error' + elif change == 'missing_case': + candidate['cases'].clear() + else: + candidate['cases']['test']['spec']['outputs'] = 1 + assert comparison_errors(base, candidate) + + +def test_comparison_preserves_direction_and_discloses_tree_count_changes(): + base, candidate = report(), report() + candidate['source']['commit'] = 'new' + candidate['cases']['test']['timing']['fit_seconds']['median'] = 1.5 + candidate['cases']['test']['memory']['fit_cupy_peak_bytes'] = 2**19 + candidate['cases']['test']['timing']['measurements'][0]['retained_trees'] = 31 + assert comparison_errors(base, candidate) == [] + text = render_comparison(base, candidate) + assert '+50.0%' in text + assert '-50.0%' in text + assert 'Retained tree counts differ' in text + + +def test_live_memory_tracking_handles_reuse_and_preexisting_allocations(): + tracker = memory_tracker(SimpleNamespace(cuda=SimpleNamespace(MemoryHook=object))) + tracker.free_postprocess(0, 512, 1, 999) # Allocated before the tracked scope. + tracker.malloc_postprocess(0, 100, 512, 1, 1) + tracker.malloc_postprocess(0, 800, 1024, 2, 2) + assert tracker.current == tracker.peak == 1536 + tracker.free_postprocess(0, 512, 1, 1) + tracker.malloc_postprocess(0, 100, 512, 1, 3) # Reused pool block. + assert tracker.current == tracker.peak == 1536 + tracker.free_postprocess(0, 1024, 2, 2) + tracker.reset_peak() + assert tracker.current == tracker.peak == 512 + tracker.free_postprocess(0, 512, 1, 3) + assert tracker.current == 0 + + +def test_real_hook_sees_private_cupy_pool(): + cp = pytest.importorskip('cupy') + if not cp.is_available(): + pytest.skip('Requires a CUDA device') + pool = cp.cuda.MemoryPool() + with memory_tracker(cp) as tracker: + with cp.cuda.using_allocator(pool.malloc): + a = cp.empty(1024, dtype=cp.float32) + b = cp.empty(1024, dtype=cp.float64) + assert tracker.current == tracker.peak == a.nbytes + b.nbytes + del a, b + assert tracker.current == 0 + pool.free_all_blocks() From b45f390e82c6b6ac7d91ebaa42261016289c0207 Mon Sep 17 00:00:00 2001 From: Rinchin Date: Fri, 30 Oct 2026 16:44:16 +0300 Subject: [PATCH 2/9] Share compilation warmup between separate timing and memory phases Keep a fresh process per case and library version while avoiding duplicate import-time Numba compilation. Preserve intermediate timing evidence if the memory phase fails. --- benchmarks/README.md | 10 +-- benchmarks/run.py | 144 ++++++++++++++++++++++--------------------- 2 files changed, 81 insertions(+), 73 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 76c275f..32e1680 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -82,13 +82,13 @@ sources may be measured in one run; compare any two results afterward: `--python /path/to/environment/bin/python` selects the worker interpreter. `--case mse-single --repeats 1` is a diagnostic subset, not the mandatory full profile. Output directories must be empty so previous evidence is not overwritten. -Workers have a 900-second timeout including setup/warmup; failures and OOM retain +Workers have a 900-second timeout including setup/warmup and both phases; failures and OOM retain logs/results, and the command exits nonzero. A version lacking a required API is reported as failed, not silently replaced by another implementation. ## Measurement contract -- Each case/measurement phase/version uses a fresh subprocess. When multiple +- Each case/version uses a fresh subprocess. When multiple versions are measured, their order alternates by case. - Each subprocess generates and hashes data, then warms one complete fit/predict. Data generation, imports, JIT warmup and estimator construction are excluded @@ -97,7 +97,9 @@ reported as failed, not silently replaced by another implementation. - Timing uses five fresh-model repetitions, `perf_counter` and device-wide CUDA synchronization. Raw samples, median and p10/p90 are retained. These are warmed timings, not cold-start latency. Holdout scoring is outside the timers. -- Memory uses a separate run so hooks/RSS sampling do not contaminate timings. +- After the timing repetitions, memory uses a separate fresh-model fit/predict in + the same subprocess. Hooks/RSS sampling are disabled during timing. Keeping + both phases in one process avoids repeating Numba import-time compilation. CuPy malloc/free hooks track **peak live rounded allocation bytes**, including Py-Boost's private fit pool. This excludes free pool cache, CUDA context and allocations outside CuPy's hooked pools; it is not total process VRAM. @@ -114,7 +116,7 @@ reported as failed, not silently replaced by another implementation. Reports identify the exact library commit and library-source checksum, benchmark implementation checksum, manifest checksum, actual dataset checksum, GPU UUID, -CPU/OS, Python, relevant dependency versions, CUDA driver/runtime, repetitions and + CPU/OS, Python, relevant dependency versions, CUDA driver/runtime, repetitions and warmups. `compare` refuses mismatched workloads, datasets, environments or failed measurements (exit code 2). Partial runs are marked `full_suite: false`; only a full five-repeat run satisfies the PR requirement. Comparison tables show measured diff --git a/benchmarks/run.py b/benchmarks/run.py index 4fd9b7c..0efaea3 100644 --- a/benchmarks/run.py +++ b/benchmarks/run.py @@ -147,60 +147,74 @@ def sync(): del warm gc.collect() cp.get_default_memory_pool().free_all_blocks() - fit_times, predict_times, measurements = [], [], [] - repeats = job['repeats'] if job['phase'] == 'timing' else 1 - for _ in range(repeats): - model, counter = new_model() - if job['phase'] == 'timing': - sync() - start = time.perf_counter() - model.fit(X, y, eval_sets=valid) - sync() - fit_times.append(time.perf_counter() - start) - start = time.perf_counter() - prediction = model.predict(test_X) - sync() - predict_times.append(time.perf_counter() - start) + def measure(phase): + fit_times, predict_times, measurements = [], [], [] + repeats = job['repeats'] if phase == 'timing' else 1 + for _ in range(repeats): + model, counter = new_model() + if phase == 'timing': + sync() + start = time.perf_counter() + model.fit(X, y, eval_sets=valid) + sync() + fit_times.append(time.perf_counter() - start) + start = time.perf_counter() + prediction = model.predict(test_X) + sync() + predict_times.append(time.perf_counter() - start) + else: + with memory_tracker(cp) as tracker: + with RssPeak(psutil.Process()) as rss_fit: + model.fit(X, y, eval_sets=valid) + sync() + fit_peak = tracker.peak + tracker.reset_peak() + with RssPeak(psutil.Process()) as rss_predict: + prediction = model.predict(test_X) + sync() + memory = {'fit_cupy_peak_bytes': fit_peak, + 'predict_cupy_peak_bytes': tracker.peak, + 'fit_rss_peak_bytes': rss_fit.peak, + 'predict_rss_peak_bytes': rss_predict.peak, + 'fit_rss_baseline_bytes': rss_fit.baseline} + if counter.count != params['ntrees']: + raise RuntimeError(f'Expected {params["ntrees"]} iterations, got {counter.count}') + if prediction.shape != (case['test'], case['outputs']): + raise RuntimeError(f'Unexpected prediction shape: {prediction.shape}') + score = quality(case['task'], test_y, prediction) + if not np.isfinite(score['value']): + raise RuntimeError('Non-finite holdout metric') + measurements.append({'trained_iterations': counter.count, + 'retained_trees': len(model.models), 'quality': score}) + cleanup(model) + del model, prediction + result = {'status': 'ok', 'environment': metadata, 'data_hash': data_hash, + 'params': params, 'measurements': measurements, + 'quality': dict(name=score['name'], **summarize([m['quality']['value'] for m in measurements])), + 'imported_library': py_boost.__file__} + if phase == 'timing': + result.update(fit_seconds=summarize(fit_times), predict_seconds=summarize(predict_times)) else: - with memory_tracker(cp) as tracker: - with RssPeak(psutil.Process()) as rss_fit: - model.fit(X, y, eval_sets=valid) - sync() - fit_peak = tracker.peak - tracker.reset_peak() - with RssPeak(psutil.Process()) as rss_predict: - prediction = model.predict(test_X) - sync() - memory = {'fit_cupy_peak_bytes': fit_peak, - 'predict_cupy_peak_bytes': tracker.peak, - 'fit_rss_peak_bytes': rss_fit.peak, - 'predict_rss_peak_bytes': rss_predict.peak, - 'fit_rss_baseline_bytes': rss_fit.baseline} - if counter.count != params['ntrees']: - raise RuntimeError(f'Expected {params["ntrees"]} iterations, got {counter.count}') - if prediction.shape != (case['test'], case['outputs']): - raise RuntimeError(f'Unexpected prediction shape: {prediction.shape}') - score = quality(case['task'], test_y, prediction) - if not np.isfinite(score['value']): - raise RuntimeError('Non-finite holdout metric') - measurements.append({'trained_iterations': counter.count, - 'retained_trees': len(model.models), 'quality': score}) - cleanup(model) - del model, prediction - result = {'status': 'ok', 'environment': metadata, 'data_hash': data_hash, - 'params': params, 'measurements': measurements, - 'quality': dict(name=score['name'], **summarize([m['quality']['value'] for m in measurements])), - 'imported_library': py_boost.__file__} - if job['phase'] == 'timing': - result.update(fit_seconds=summarize(fit_times), predict_seconds=summarize(predict_times)) - else: - result.update(memory) - return result + result.update(memory) + return result + + result = {'status': 'running', 'environment': metadata, 'data_hash': data_hash} + for phase in ['timing', 'memory']: + try: + result[phase] = measure(phase) + except Exception: + result[phase] = {'status': 'error', 'error': traceback.format_exc()} + result['status'] = 'error' + return result + write_json(job['result_path'], result) + result['status'] = 'ok' + return result def invoke(python, output, job, timeout): output = Path(output) request, result, log = output.with_suffix('.job.json'), output.with_suffix('.json'), output.with_suffix('.log') + job['result_path'] = str(result) write_json(request, job) env = dict(os.environ) env['CUDA_VISIBLE_DEVICES'] = job['device'] @@ -221,7 +235,8 @@ def invoke(python, output, job, timeout): if process.returncode and record.get('status') == 'ok': record = {'status': 'error', 'error': f'Worker exited {process.returncode}'} except subprocess.TimeoutExpired: - record = {'status': 'error', 'error': f'Timeout after {timeout} seconds'} + record = json.loads(result.read_text()) if result.exists() else {} + record.update(status='error', error=f'Timeout after {timeout} seconds') record['log'] = str(log) return record @@ -265,28 +280,19 @@ def run(args): directories.append(directory) write_json(directory / 'results.json', report) for index, case in enumerate(cases): - # Alternate version order for each case; each case/phase gets a fresh process. + # One fresh process per version/case; timing and memory are separate phases. order = list(range(len(sources))) if index % 2: order.reverse() - for phase in ['timing', 'memory']: - for i in order: - print(f'{sources[i]["commit"][:12]} {case["id"]} {phase}', flush=True) - job = dict(source=sources[i], phase=phase, device=args.device, suite=suite, - case=case, repeats=args.repeats) - record = invoke(args.python, directories[i] / 'raw' / f'{case["id"]}-{phase}', job, args.timeout) - entry = reports[i]['cases'].setdefault(case['id'], {'spec': case, 'status': 'running'}) - entry[phase] = record - if record['status'] == 'ok' and record['environment'] != reports[i]['environment']: - record.update(status='error', error='Environment changed during the run') - if phase == 'memory': - a, b = entry['timing'], entry['memory'] - entry['status'] = 'ok' if a['status'] == b['status'] == 'ok' else 'error' - if entry['status'] == 'ok': - entry['data_hash'] = a['data_hash'] - if a['data_hash'] != b['data_hash']: - entry.update(status='error', error='Dataset changed between measurement phases') - write_json(directories[i] / 'results.json', reports[i]) + for i in order: + print(f'{sources[i]["commit"][:12]} {case["id"]}', flush=True) + job = dict(source=sources[i], phase='case', device=args.device, suite=suite, + case=case, repeats=args.repeats) + record = invoke(args.python, directories[i] / 'raw' / case['id'], job, args.timeout) + if record['status'] == 'ok' and record['environment'] != reports[i]['environment']: + record.update(status='error', error='Environment changed during the run') + reports[i]['cases'][case['id']] = dict(spec=case, **record) + write_json(directories[i] / 'results.json', reports[i]) for report, directory in zip(reports, directories): report['complete'] = all(c['status'] == 'ok' for c in report['cases'].values()) report['finished_utc'] = datetime.datetime.now(datetime.timezone.utc).isoformat() @@ -352,7 +358,7 @@ def main(): try: result = worker(json.loads(Path(args.job).read_text())) write_json(args.result, result) - return 0 + return 0 if result['status'] == 'ok' else 1 except Exception: write_json(args.result, {'status': 'error', 'error': traceback.format_exc()}) traceback.print_exc() From d3e74a9f1643483bb3862a11f389c9e22e79c9f7 Mon Sep 17 00:00:00 2001 From: Rinchin Date: Fri, 30 Oct 2026 17:05:55 +0300 Subject: [PATCH 3/9] Expose single-GPU hardware and sketch width in benchmark reports Record and display one-device execution, physical/CUDA-visible VRAM, GPU UUID, PCI bus, SM count, memory bus, power limit, maximum SM clock and driver versions. Show target dimension and sketch width per case, distinguishing projections from sampling with replacement. Add GPU-count and report regressions. --- AGENTS.md | 3 ++ benchmarks/README.md | 60 +++++++++++++++++++++----------- benchmarks/run.py | 42 ++++++++++++++++++---- tests/test_benchmark_protocol.py | 54 ++++++++++++++++++++++++++-- 4 files changed, 130 insertions(+), 29 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 07018f4..852633f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,6 +14,9 @@ with a single sketch method, a small-output case, or an improvised timing script checksums. Verify that the imported library comes from the intended checkout. - Run `benchmarks/run.py run` with both source paths, an explicit device and a new output directory. A missing GPU, failed case, timeout or OOM is not a pass. +- Use exactly one physical GPU. Never run benchmark cases or versions in parallel + on multiple GPUs. Publish its model, VRAM, UUID, compute capability and driver + versions in the report, plus full output count K and sketch width k per case. - Use `benchmarks/run.py compare` to produce the report; do not compare mismatched environments/workloads. Inspect time, peak live CuPy bytes, CPU RSS, holdout quality, executed iterations and retained tree counts. diff --git a/benchmarks/README.md b/benchmarks/README.md index 32e1680..a37b7ca 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -10,25 +10,36 @@ of the directory containing the harness. Definitions are in `suite-v1.json`. All cases build **32 iterations**, use learning rate 0.1, seed 42, full row/column sampling, and `use_hess=True`. Defaults are depth -4 and 64 bins. Sketch cases keep 8 outputs for structure search. Early stopping is -disabled. The benchmark counts executed iterations and separately records retained +4 and 64 bins. Every sketch case explicitly sets **`sketch_outputs=8`**. +Here **K** is the original number of outputs and **k=8** is the sketch width used +only for structure search; final predictions still contain all K outputs: + +- `proj`: eight projected components combining the original outputs. +- `rand`: eight output draws with replacement, so at most eight distinct outputs. +- `filter`: eight importance-weighted output draws with replacement, so at most + eight distinct outputs. +- `topk`: eight distinct outputs with the largest gradient norms. +- Full `GradientBoosting`: all K outputs, without a sketch. + +Early stopping is disabled. The benchmark counts executed iterations and +separately records retained trees: existing Py-Boost versions can prune to the best validation iteration even with `es=0`. Different retained tree counts are disclosed in comparisons. -| Case | Train / validation / test rows | Features | Outputs | Model | -|---|---:|---:|---:|---| -| mse-single | 16384 / 4096 / 4096 | 32 | 1 | GradientBoosting | -| mse-full-128 | 8192 / 2048 / 2048 | 32 | 128 | GradientBoosting | -| mse-proj-32 | 16384 / 4096 / 4096 | 32 | 32 | SketchBoost/proj | -| mse-proj-128 | 16384 / 4096 / 4096 | 32 | 128 | SketchBoost/proj | -| mse-proj-1024 | 16384 / 4096 / 4096 | 32 | 1024 | SketchBoost/proj | -| mse-proj-10000 | 16384 / 4096 / 4096 | 32 | 10000 | SketchBoost/proj | -| bce-rand-1024 | 16384 / 4096 / 4096 | 32 | 1024 | SketchBoost/rand | -| bce-filter-1024 | 16384 / 4096 / 4096 | 32 | 1024 | SketchBoost/filter | -| mse-topk-10000 | 4096 / 1024 / 1024 | 32 | 10000 | SketchBoost/topk | -| ce-proj-512 | 16384 / 4096 / 4096 | 32 | 512 classes | SketchBoost/proj | -| validation-proj-1024 | 8192 / 32768 / 4096 | 32 | 1024 | SketchBoost/proj; two validation sets | -| histogram-topk-128 | 8192 / 2048 / 2048 | 128 | 128 | SketchBoost/topk; depth 8, 256 bins | +| Case | Train / validation / test rows | Features | Outputs K | Sketch width k | Model | +|---|---:|---:|---:|---:|---| +| mse-single | 16384 / 4096 / 4096 | 32 | 1 | all | GradientBoosting | +| mse-full-128 | 8192 / 2048 / 2048 | 32 | 128 | all | GradientBoosting | +| mse-proj-32 | 16384 / 4096 / 4096 | 32 | 32 | 8 | SketchBoost/proj | +| mse-proj-128 | 16384 / 4096 / 4096 | 32 | 128 | 8 | SketchBoost/proj | +| mse-proj-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 8 | SketchBoost/proj | +| mse-proj-10000 | 16384 / 4096 / 4096 | 32 | 10000 | 8 | SketchBoost/proj | +| bce-rand-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 8 | SketchBoost/rand | +| bce-filter-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 8 | SketchBoost/filter | +| mse-topk-10000 | 4096 / 1024 / 1024 | 32 | 10000 | 8 | SketchBoost/topk | +| ce-proj-512 | 16384 / 4096 / 4096 | 32 | 512 classes | 8 | SketchBoost/proj | +| validation-proj-1024 | 8192 / 32768 / 4096 | 32 | 1024 | 8 | SketchBoost/proj; two validation sets | +| histogram-topk-128 | 8192 / 2048 / 2048 | 128 | 128 | 8 | SketchBoost/topk; depth 8, 256 bins | The four `mse-proj-*` cases change only the number of outputs. Other cases exercise full-output scoring, separate/shared Hessians, wide targets, validation memory, @@ -63,13 +74,22 @@ git worktree add --detach ../Py-Boost-v0.5.2 v0.5.2 ``` Commit changes to `py_boost/` or `pyproject.toml` first: dirty library sources are -rejected. Use a physical GPU index or UUID (`nvidia-smi -L`). Do not share that GPU -with another workload. Preflight performs a real allocation/kernel, rejects other +rejected. **Exactly one physical GPU is used for the entire run.** Use a physical +GPU index or UUID (`nvidia-smi -L`). `--device 1` selects only physical GPU #1; it +is exposed inside each worker as logical `cuda:0` through `CUDA_VISIBLE_DEVICES`. +A comma-separated GPU list is rejected, and workers verify that exactly one CUDA +device is visible. Cases, versions and measurement phases run sequentially. +Do not share the selected GPU with another workload. Preflight performs a real allocation/kernel, rejects other compute processes on the GPU, and records its UUID. CPU thread counts are fixed at 4. No GPU is a failure, not a skipped or CPU-emulated benchmark. Each source produces `//results.json` and raw worker -requests/results/logs. Two-source runs also produce `comparison.md`. More than two +requests/results/logs. GPU count, model, physical and CUDA-visible VRAM, UUID, +PCI bus, compute capability, SM count, memory bus width, power limit, maximum SM +clock and NVIDIA/CUDA driver/runtime versions are recorded +in JSON and displayed in the comparison report. Each comparison row shows the +full output count K and sketch width k/method explicitly. +Two-source runs also produce `comparison.md`. More than two sources may be measured in one run; compare any two results afterward: ```bash @@ -116,7 +136,7 @@ reported as failed, not silently replaced by another implementation. Reports identify the exact library commit and library-source checksum, benchmark implementation checksum, manifest checksum, actual dataset checksum, GPU UUID, - CPU/OS, Python, relevant dependency versions, CUDA driver/runtime, repetitions and +CPU/OS, Python, relevant dependency versions, CUDA driver/runtime, repetitions and warmups. `compare` refuses mismatched workloads, datasets, environments or failed measurements (exit code 2). Partial runs are marked `full_suite: false`; only a full five-repeat run satisfies the PR requirement. Comparison tables show measured diff --git a/benchmarks/run.py b/benchmarks/run.py index 0efaea3..22a52d7 100644 --- a/benchmarks/run.py +++ b/benchmarks/run.py @@ -59,8 +59,14 @@ def environment(cp, np, psutil, device): cp.zeros(1).sum().item() # A real allocation and kernel; no CPU fallback. props = cp.cuda.runtime.getDeviceProperties(0) name = props['name'] - uuid = subprocess.check_output( - ['nvidia-smi', '-i', device, '--query-gpu=uuid', '--format=csv,noheader'], text=True).strip() + gpu_details = subprocess.check_output( + ['nvidia-smi', '-i', device, + '--query-gpu=uuid,driver_version,pci.bus_id,memory.total,power.limit,clocks.max.sm', + '--format=csv,noheader,nounits'], text=True).strip() + if '\n' in gpu_details: + raise RuntimeError('GPU selector resolved to more than one physical device') + uuid, driver_version, pci_bus_id, memory_mib, power_limit, max_sm_clock = [ + part.strip() for part in gpu_details.split(',')] processes = subprocess.check_output( ['nvidia-smi', '--query-compute-apps=gpu_uuid,pid', '--format=csv,noheader,nounits'], text=True) @@ -83,9 +89,14 @@ def environment(cp, np, psutil, device): Path('/proc/cpuinfo').read_text().splitlines() if line.startswith('model name')), cpu) return {'python': platform.python_version(), 'platform': platform.platform(), 'cpu': cpu, 'cpu_threads': int(os.environ['NUMBA_NUM_THREADS']), - 'host_ram_bytes': psutil.virtual_memory().total, + 'host_ram_bytes': psutil.virtual_memory().total, 'gpu_count': 1, 'gpu': {'name': name.decode() if isinstance(name, bytes) else name, 'uuid': uuid, 'vram_bytes': props['totalGlobalMem'], + 'driver_version': driver_version, 'pci_bus_id': pci_bus_id, + 'physical_vram_bytes': int(float(memory_mib) * 2**20), + 'power_limit_watts': power_limit, 'max_sm_clock_mhz': max_sm_clock, + 'multiprocessors': props['multiProcessorCount'], + 'memory_bus_width_bits': props.get('memoryBusWidth'), 'compute_capability': [props['major'], props['minor']]}, 'cuda_driver': cp.cuda.runtime.driverGetVersion(), 'cuda_runtime': cp.cuda.runtime.runtimeGetVersion(), 'packages': packages} @@ -308,11 +319,25 @@ def render_comparison(base, head): f'Base: `{base["source"]["commit"]}`; candidate: `{head["source"]["commit"]}`.', ''] if errors: return '\n'.join(lines + ['**Not comparable:**'] + [f'- {e}' for e in errors]) + '\n' - lines += ['Times are warmed wall-clock medians. GPU memory is peak live CuPy allocations,', + env = base['environment'] + gpu = env['gpu'] + capability = '.'.join(str(part) for part in gpu['compute_capability']) + def cuda_version(value): + return f'{value // 1000}.{value % 1000 // 10}' + lines += [f'**GPU count: {env.get("gpu_count", "not recorded")}**; versions and cases execute sequentially.', + f'GPU: **{gpu["name"]}**; physical VRAM: **{gpu.get("physical_vram_bytes", gpu["vram_bytes"]) / 2**30:.1f} GiB**; CUDA-visible: **{gpu["vram_bytes"] / 2**30:.1f} GiB**; compute capability: **{capability}**.', + f'UUID: `{gpu["uuid"]}`; PCI bus: `{gpu.get("pci_bus_id", "not recorded")}`.', + f'SMs: {gpu.get("multiprocessors", "not recorded")}; memory bus: {gpu.get("memory_bus_width_bits", "not recorded")} bits; NVIDIA driver: {gpu.get("driver_version", "not recorded")}.', + f'Power limit: {gpu.get("power_limit_watts", "not recorded")} W; max SM clock: {gpu.get("max_sm_clock_mhz", "not recorded")} MHz.', + f'CUDA driver API: {cuda_version(env["cuda_driver"])}; CUDA runtime: {cuda_version(env["cuda_runtime"])}.', '', + 'K is the full target dimension; k is the sketch width used for structure search.', + '`proj`: k projected components; `rand`/`filter`: k draws with replacement (at most k distinct outputs);', + '`topk`: k distinct outputs. Full GradientBoosting uses all K outputs. Predictions always have K outputs.', '', + 'Times are warmed wall-clock medians. GPU memory is peak live CuPy allocations,', 'including the private fit pool; CUDA context, pool cache and external library allocations are excluded.', 'Quality is a holdout metric (lower is better). These are measured changes, not statistical verdicts.', '', - '| Case | Fit s (base → PR) | Δ fit | Predict s (base → PR) | Δ predict | Fit MiB (base → PR) | Δ memory | Quality (base → PR) |', - '|---|---:|---:|---:|---:|---:|---:|---:|'] + '| Case | Outputs K | Sketch k | Fit s (base → PR) | Δ fit | Predict s (base → PR) | Δ predict | Fit MiB (base → PR) | Δ memory | Quality (base → PR) |', + '|---|---:|---|---:|---:|---:|---:|---:|---:|---:|'] notes = [] for case_id in base['case_ids']: a, b = base['cases'][case_id], head['cases'][case_id] @@ -327,7 +352,10 @@ def pair(section, key, scale=1): mem, dm = pair('memory', 'fit_cupy_peak_bytes', 2**20) qa = a['timing']['quality']['median'] qb = b['timing']['quality']['median'] - lines.append(f'| {case_id} | {fit} | {df} | {pred} | {dp} | {mem} | {dm} | {qa:.6g} → {qb:.6g} |') + spec = a['spec'] + sketch = spec.get('params', {}).get('sketch_method') + sketch_width = f'{spec["params"]["sketch_outputs"]} ({sketch})' if sketch else f'all {spec["outputs"]}' + lines.append(f'| {case_id} | {spec["outputs"]} | {sketch_width} | {fit} | {df} | {pred} | {dp} | {mem} | {dm} | {qa:.6g} → {qb:.6g} |') ra = [m['retained_trees'] for m in a['timing']['measurements']] rb = [m['retained_trees'] for m in b['timing']['measurements']] if len(set(ra + rb)) > 1: diff --git a/tests/test_benchmark_protocol.py b/tests/test_benchmark_protocol.py index 2cf0b0c..5212929 100644 --- a/tests/test_benchmark_protocol.py +++ b/tests/test_benchmark_protocol.py @@ -18,7 +18,8 @@ from pyboost_benchmark_suite.gpu import memory_tracker from pyboost_benchmark_suite.protocol import comparison_errors, load_suite, make_data, quality -from pyboost_benchmark_suite.run import render_comparison +from pyboost_benchmark_suite.run import environment, render_comparison +from pyboost_benchmark_suite import run as runner def small_case(task='regression'): @@ -69,7 +70,13 @@ def report(): quality={'median': .2}, measurements=[{'retained_trees': 32}]) return dict(schema_version=1, benchmark_id='pyboost-gpu-v1', implementation_hash='abc', suite_hash='def', case_ids=['test'], repeats=5, warmups=1, complete=True, - environment={'gpu': 'A100', 'packages': {'numpy': '2'}}, + environment={'gpu_count': 1, 'gpu': {'name': 'NVIDIA A100 80GB PCIe', + 'uuid': 'GPU-test', 'vram_bytes': 80 * 2**30, 'compute_capability': [8, 0], + 'physical_vram_bytes': 80 * 2**30, 'power_limit_watts': '300', + 'max_sm_clock_mhz': '1410', + 'pci_bus_id': '00000000:14:00.0', 'multiprocessors': 108, + 'memory_bus_width_bits': 5120, 'driver_version': '550.163.01'}, + 'cuda_driver': 12040, 'cuda_runtime': 12090, 'packages': {'numpy': '2'}}, source={'commit': 'abc'}, cases={'test': dict(status='ok', data_hash='123', spec={'outputs': 1024}, timing=timing, memory={'fit_cupy_peak_bytes': 2**20})}) @@ -114,6 +121,49 @@ def test_comparison_preserves_direction_and_discloses_tree_count_changes(): assert 'Retained tree counts differ' in text +def test_report_displays_single_gpu_characteristics(): + text = render_comparison(report(), report()) + for value in ['GPU count: 1', 'NVIDIA A100 80GB PCIe', '80.0 GiB', + 'GPU-test', '8.0', '108', '5120', '550.163.01', '12.4', '12.9', + '300 W', '1410 MHz']: + assert value in text + + +@pytest.mark.parametrize('method', ['proj', 'rand', 'filter', 'topk']) +def test_report_distinguishes_target_dimension_from_sketch_width(method): + base, candidate = report(), report() + for result in [base, candidate]: + result['cases']['test']['spec'] = dict(outputs=10000, params=dict( + sketch_method=method, sketch_outputs=8)) + text = render_comparison(base, candidate) + assert '| test | 10000 | 8 (' + method + ') |' in text + assert 'at most k distinct outputs' in text + assert 'Predictions always have K outputs' in text + + +def test_report_shows_full_output_scoring_without_sketch(): + text = render_comparison(report(), report()) + assert '| test | 1024 | all 1024 |' in text + + +@pytest.mark.parametrize('count', [0, 2]) +def test_preflight_rejects_any_visible_gpu_count_other_than_one(count): + cp = SimpleNamespace(cuda=SimpleNamespace(runtime=SimpleNamespace(getDeviceCount=lambda: count))) + with pytest.raises(RuntimeError, match='Exactly one visible CUDA device'): + environment(cp, np, None, '0') + + +def test_cli_rejects_multiple_gpu_ids_before_starting_workers(monkeypatch, tmp_path): + monkeypatch.setattr(runner, 'source_info', lambda path: dict(commit='abc', path=str(path))) + monkeypatch.setattr(runner, 'invoke', lambda *args: pytest.fail('Worker must not start')) + output = tmp_path / 'results' + args = SimpleNamespace(source=['checkout'], case=None, repeats=5, timeout=900, + device='0,1', output=str(output)) + with pytest.raises(ValueError, match='exactly one GPU'): + runner.run(args) + assert not output.exists() + + def test_live_memory_tracking_handles_reuse_and_preexisting_allocations(): tracker = memory_tracker(SimpleNamespace(cuda=SimpleNamespace(MemoryHook=object))) tracker.free_postprocess(0, 512, 1, 999) # Allocated before the tracked scope. From 0c9060c2e8be01af876d95c8aba4a88e9c635cef Mon Sep 17 00:00:00 2001 From: Rinchin Date: Fri, 30 Oct 2026 17:07:52 +0300 Subject: [PATCH 4/9] Resolve the selected physical GPU to UUID before CUDA initialization Avoid mismatched device identity when CUDA and nvidia-smi ordinal ordering differ. Reject selectors resolving to multiple cards or MIG instances. --- benchmarks/README.md | 4 +++- benchmarks/run.py | 15 +++++++++++++-- tests/test_benchmark_protocol.py | 17 +++++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index a37b7ca..a95750c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -76,7 +76,9 @@ git worktree add --detach ../Py-Boost-v0.5.2 v0.5.2 Commit changes to `py_boost/` or `pyproject.toml` first: dirty library sources are rejected. **Exactly one physical GPU is used for the entire run.** Use a physical GPU index or UUID (`nvidia-smi -L`). `--device 1` selects only physical GPU #1; it -is exposed inside each worker as logical `cuda:0` through `CUDA_VISIBLE_DEVICES`. +is resolved to its physical UUID and exposed inside each worker as logical +`cuda:0` through `CUDA_VISIBLE_DEVICES`. UUID selection avoids differences between +CUDA and `nvidia-smi` ordinal ordering. A comma-separated GPU list is rejected, and workers verify that exactly one CUDA device is visible. Cases, versions and measurement phases run sequentially. Do not share the selected GPU with another workload. Preflight performs a real allocation/kernel, rejects other diff --git a/benchmarks/run.py b/benchmarks/run.py index 22a52d7..dd856f6 100644 --- a/benchmarks/run.py +++ b/benchmarks/run.py @@ -53,6 +53,16 @@ def source_info(source): 'library_hash': digest.hexdigest()} +def resolve_gpu(selector): + """Use a physical UUID: CUDA and nvidia-smi ordinal ordering can differ.""" + devices = subprocess.check_output( + ['nvidia-smi', '-i', selector, '--query-gpu=uuid', '--format=csv,noheader'], + text=True).strip().splitlines() + if len(devices) != 1 or not devices[0].startswith('GPU-'): + raise ValueError('Select exactly one physical GPU index or UUID') + return devices[0] + + def environment(cp, np, psutil, device): if cp.cuda.runtime.getDeviceCount() != 1: raise RuntimeError('Exactly one visible CUDA device is required') @@ -264,6 +274,7 @@ def run(args): raise ValueError('Repeats and timeout must be positive') if ',' in args.device or not args.device: raise ValueError('Select exactly one GPU index or UUID') + device = resolve_gpu(args.device) cases = [c for c in suite['cases'] if c['id'] in selected] output = Path(args.output).resolve() output.mkdir(parents=True, exist_ok=True) @@ -280,7 +291,7 @@ def run(args): 'full_suite': len(cases) == len(suite['cases']), 'repeats': args.repeats, 'warmups': suite['warmups'], 'suite': suite, 'cases': {}, 'complete': False} - job = dict(source=source, phase='metadata', device=args.device, suite=suite) + job = dict(source=source, phase='metadata', device=device, suite=suite) probe = invoke(args.python, directory / 'raw' / 'metadata', job, args.timeout) if probe['status'] != 'ok': report['error'] = probe @@ -297,7 +308,7 @@ def run(args): order.reverse() for i in order: print(f'{sources[i]["commit"][:12]} {case["id"]}', flush=True) - job = dict(source=sources[i], phase='case', device=args.device, suite=suite, + job = dict(source=sources[i], phase='case', device=device, suite=suite, case=case, repeats=args.repeats) record = invoke(args.python, directories[i] / 'raw' / case['id'], job, args.timeout) if record['status'] == 'ok' and record['environment'] != reports[i]['environment']: diff --git a/tests/test_benchmark_protocol.py b/tests/test_benchmark_protocol.py index 5212929..3541758 100644 --- a/tests/test_benchmark_protocol.py +++ b/tests/test_benchmark_protocol.py @@ -164,6 +164,23 @@ def test_cli_rejects_multiple_gpu_ids_before_starting_workers(monkeypatch, tmp_p assert not output.exists() +def test_gpu_index_is_resolved_to_physical_uuid(monkeypatch): + commands = [] + def query(command, **kwargs): + commands.append(command) + return 'GPU-selected\n' + monkeypatch.setattr(runner.subprocess, 'check_output', query) + assert runner.resolve_gpu('1') == 'GPU-selected' + assert commands == [['nvidia-smi', '-i', '1', '--query-gpu=uuid', '--format=csv,noheader']] + + +@pytest.mark.parametrize('response', ['', 'GPU-a\nGPU-b\n', 'MIG-instance\n']) +def test_gpu_selector_must_resolve_to_one_physical_card(monkeypatch, response): + monkeypatch.setattr(runner.subprocess, 'check_output', lambda *args, **kwargs: response) + with pytest.raises(ValueError, match='exactly one physical GPU'): + runner.resolve_gpu('selector') + + def test_live_memory_tracking_handles_reuse_and_preexisting_allocations(): tracker = memory_tracker(SimpleNamespace(cuda=SimpleNamespace(MemoryHook=object))) tracker.free_postprocess(0, 512, 1, 999) # Allocated before the tracked scope. From 4da985bd650771618947357b749721e5fdba0700 Mon Sep 17 00:00:00 2001 From: Rinchin Date: Fri, 30 Oct 2026 17:35:36 +0300 Subject: [PATCH 5/9] Add a separate frozen k=5 GPU benchmark profile Select suite manifests explicitly while preserving the original k=8 workload. Fingerprint the selected suite and verify that the k=5 variant changes only sketch width and benchmark identity. --- benchmarks/README.md | 11 ++ benchmarks/protocol.py | 9 +- benchmarks/run.py | 6 +- benchmarks/suite-v1-k5.json | 231 +++++++++++++++++++++++++++++++ tests/test_benchmark_protocol.py | 24 +++- 5 files changed, 273 insertions(+), 8 deletions(-) create mode 100644 benchmarks/suite-v1-k5.json diff --git a/benchmarks/README.md b/benchmarks/README.md index a95750c..cf51dcd 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -73,6 +73,17 @@ git worktree add --detach ../Py-Boost-v0.5.2 v0.5.2 --device 1 --output .benchmark-results/v052-vs-current ``` +For the same 12 workloads with `sketch_outputs=5`, select the separate frozen +`pyboost-gpu-v1-k5` profile. The k=8 manifest remains unchanged; reports with +different sketch widths are not comparable: + +```bash +.venv-benchmark/bin/python benchmarks/run.py run \ + --suite benchmarks/suite-v1-k5.json \ + --source /path/to/baseline --source /path/to/candidate \ + --device 1 --output .benchmark-results/k5-comparison +``` + Commit changes to `py_boost/` or `pyproject.toml` first: dirty library sources are rejected. **Exactly one physical GPU is used for the entire run.** Use a physical GPU index or UUID (`nvidia-smi -L`). `--device 1` selects only physical GPU #1; it diff --git a/benchmarks/protocol.py b/benchmarks/protocol.py index eff4d27..6c3ae44 100644 --- a/benchmarks/protocol.py +++ b/benchmarks/protocol.py @@ -14,16 +14,17 @@ def digest_json(value): return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest() -def implementation_hash(): +def implementation_hash(suite=None): digest = hashlib.sha256() - for name in ['suite-v1.json', 'protocol.py', 'gpu.py', 'run.py']: + digest.update(digest_json(load_suite() if suite is None else suite).encode()) + for name in ['protocol.py', 'gpu.py', 'run.py']: digest.update(name.encode()) digest.update((ROOT / name).read_bytes()) return digest.hexdigest() -def load_suite(): - return json.loads((ROOT / 'suite-v1.json').read_text()) +def load_suite(path=None): + return json.loads((Path(path) if path is not None else ROOT / 'suite-v1.json').read_text()) def make_data(case, seed): diff --git a/benchmarks/run.py b/benchmarks/run.py index dd856f6..e31103f 100644 --- a/benchmarks/run.py +++ b/benchmarks/run.py @@ -263,7 +263,7 @@ def invoke(python, output, job, timeout): def run(args): - suite = load_suite() + suite = load_suite(args.suite) sources = [source_info(p) for p in args.source] if len({s['commit'] for s in sources}) != len(sources): raise ValueError('Each source must have a different commit') @@ -285,7 +285,7 @@ def run(args): directory = output / source['commit'][:12] (directory / 'raw').mkdir(parents=True) report = {'schema_version': SCHEMA_VERSION, 'benchmark_id': suite['benchmark_id'], - 'implementation_hash': implementation_hash(), 'suite_hash': digest_json(suite), + 'implementation_hash': implementation_hash(suite), 'suite_hash': digest_json(suite), 'started_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'source': source, 'case_ids': [c['id'] for c in cases], 'full_suite': len(cases) == len(suite['cases']), @@ -379,6 +379,8 @@ def main(): commands = parser.add_subparsers(dest='command', required=True) command = commands.add_parser('run', help='Run one or more clean source checkouts') command.add_argument('--source', action='append', required=True, help='Git checkout; repeat to compare versions') + command.add_argument('--suite', type=Path, default=ROOT / 'suite-v1.json', + help='Frozen suite manifest (default: suite-v1.json, k=8)') command.add_argument('--output', required=True) command.add_argument('--device', required=True, help='One physical GPU index or UUID') command.add_argument('--python', default=sys.executable, help='Interpreter containing the pinned dependencies') diff --git a/benchmarks/suite-v1-k5.json b/benchmarks/suite-v1-k5.json new file mode 100644 index 0000000..4b9cf1c --- /dev/null +++ b/benchmarks/suite-v1-k5.json @@ -0,0 +1,231 @@ +{ + "benchmark_id": "pyboost-gpu-v1-k5", + "data_seed": 20261030, + "model_seed": 42, + "cpu_threads": 4, + "warmups": 1, + "repeats": 5, + "sketch_params": { + "proj": { + "norm": true + }, + "rand": { + "smooth": 0.1, + "replace": true + }, + "filter": { + "sample": true, + "smooth": 0.1, + "ntrees": 1 + }, + "topk": {} + }, + "common_params": { + "ntrees": 32, + "max_depth": 4, + "max_bin": 64, + "lr": 0.1, + "es": 0, + "verbose": 1000000, + "min_data_in_leaf": 10, + "colsample": 1.0, + "subsample": 1.0, + "use_hess": true, + "lambda_l2": 1.0, + "gd_steps": 1, + "min_gain_to_split": 0, + "quantization": "Quantile", + "quant_sample": 2000000, + "min_data_in_bin": 3 + }, + "cases": [ + { + "id": "mse-single", + "task": "regression", + "model": "GradientBoosting", + "loss": "mse", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 1, + "validation_sets": 1, + "params": {} + }, + { + "id": "mse-full-128", + "task": "regression", + "model": "GradientBoosting", + "loss": "mse", + "train": 8192, + "valid": 2048, + "test": 2048, + "features": 32, + "outputs": 128, + "validation_sets": 1, + "params": {} + }, + { + "id": "mse-proj-32", + "task": "regression", + "model": "SketchBoost", + "loss": "mse", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 32, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "mse-proj-128", + "task": "regression", + "model": "SketchBoost", + "loss": "mse", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 128, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "mse-proj-1024", + "task": "regression", + "model": "SketchBoost", + "loss": "mse", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 1024, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "mse-proj-10000", + "task": "regression", + "model": "SketchBoost", + "loss": "mse", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 10000, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "bce-rand-1024", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 1024, + "validation_sets": 1, + "params": { + "sketch_method": "rand", + "sketch_outputs": 5 + } + }, + { + "id": "bce-filter-1024", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 1024, + "validation_sets": 1, + "params": { + "sketch_method": "filter", + "sketch_outputs": 5 + } + }, + { + "id": "mse-topk-10000", + "task": "regression", + "model": "SketchBoost", + "loss": "mse", + "train": 4096, + "valid": 1024, + "test": 1024, + "features": 32, + "outputs": 10000, + "validation_sets": 1, + "params": { + "sketch_method": "topk", + "sketch_outputs": 5 + } + }, + { + "id": "ce-proj-512", + "task": "multiclass", + "model": "SketchBoost", + "loss": "crossentropy", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 512, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "validation-proj-1024", + "task": "regression", + "model": "SketchBoost", + "loss": "mse", + "train": 8192, + "valid": 32768, + "test": 4096, + "features": 32, + "outputs": 1024, + "validation_sets": 2, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "histogram-topk-128", + "task": "regression", + "model": "SketchBoost", + "loss": "mse", + "train": 8192, + "valid": 2048, + "test": 2048, + "features": 128, + "outputs": 128, + "validation_sets": 1, + "params": { + "sketch_method": "topk", + "sketch_outputs": 5, + "max_depth": 8, + "max_bin": 256 + } + } + ] +} diff --git a/tests/test_benchmark_protocol.py b/tests/test_benchmark_protocol.py index 3541758..bab357c 100644 --- a/tests/test_benchmark_protocol.py +++ b/tests/test_benchmark_protocol.py @@ -17,7 +17,7 @@ _spec.loader.exec_module(_package) from pyboost_benchmark_suite.gpu import memory_tracker -from pyboost_benchmark_suite.protocol import comparison_errors, load_suite, make_data, quality +from pyboost_benchmark_suite.protocol import ROOT, comparison_errors, implementation_hash, load_suite, make_data, quality from pyboost_benchmark_suite.run import environment, render_comparison from pyboost_benchmark_suite import run as runner @@ -55,6 +55,26 @@ def test_same_multilabel_data_for_different_sketch_methods(): assert make_data(rand, suite['data_seed'])[-1] == make_data(filt, suite['data_seed'])[-1] +def test_k5_profile_changes_only_sketch_width_and_identity(): + from copy import deepcopy + original = load_suite() + k5 = load_suite(ROOT / 'suite-v1-k5.json') + assert k5['benchmark_id'] == 'pyboost-gpu-v1-k5' + expected = deepcopy(original) + expected['benchmark_id'] = k5['benchmark_id'] + for case in expected['cases']: + if case['model'] == 'SketchBoost': + case['params']['sketch_outputs'] = 5 + assert k5 == expected + assert implementation_hash(k5) != implementation_hash(original) + assert all(c['params']['sketch_outputs'] == 8 for c in original['cases'] + if c['model'] == 'SketchBoost') + before, after = small_case(), small_case() + before['params'] = dict(sketch_outputs=8) + after['params'] = dict(sketch_outputs=5) + assert make_data(before, 42)[-1] == make_data(after, 42)[-1] + + def test_holdout_metrics_and_invalid_predictions(): assert quality('multilabel', np.array([[0, 1]], dtype=np.float32), np.array([[.5, .5]], dtype=np.float32))['value'] == pytest.approx(np.log(2)) @@ -157,7 +177,7 @@ def test_cli_rejects_multiple_gpu_ids_before_starting_workers(monkeypatch, tmp_p monkeypatch.setattr(runner, 'source_info', lambda path: dict(commit='abc', path=str(path))) monkeypatch.setattr(runner, 'invoke', lambda *args: pytest.fail('Worker must not start')) output = tmp_path / 'results' - args = SimpleNamespace(source=['checkout'], case=None, repeats=5, timeout=900, + args = SimpleNamespace(source=['checkout'], suite=None, case=None, repeats=5, timeout=900, device='0,1', output=str(output)) with pytest.raises(ValueError, match='exactly one GPU'): runner.run(args) From 3eb0e2a1bb15e5ff5726fdf3912b8bdf88aec1e6 Mon Sep 17 00:00:00 2001 From: Rinchin Date: Sat, 31 Oct 2026 00:14:13 +0300 Subject: [PATCH 6/9] Default local GPU benchmarks to five sketch outputs Select the frozen k=5 profile by default in the loader and CLI, update repository instructions and documentation, and retain the explicit k=8 profile for historical comparisons. --- AGENTS.md | 5 +-- benchmarks/README.md | 52 ++++++++++++++++---------------- benchmarks/protocol.py | 3 +- benchmarks/run.py | 8 ++--- tests/test_benchmark_protocol.py | 17 ++++++++++- 5 files changed, 51 insertions(+), 34 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 852633f..d937d97 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,10 +4,11 @@ ## Required local GPU benchmark -Before declaring a PR ready, run the complete `pyboost-gpu-v1` benchmark described +Before declaring a PR ready, run the complete `pyboost-gpu-v1-k5` benchmark described in `benchmarks/README.md` against the PR's exact base and candidate commits, using the same harness, pinned environment and idle GPU. Use all 12 cases, one warmup -and five timing repetitions; keep memory runs separate. Do not replace the suite +and five timing repetitions, with `sketch_outputs=5` in every sketch case; keep +memory runs separate. Do not replace the suite with a single sketch method, a small-output case, or an improvised timing script. - Commit library changes first. Record both code SHAs and the benchmark/data diff --git a/benchmarks/README.md b/benchmarks/README.md index cf51dcd..09f4937 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,6 +1,6 @@ -# Py-Boost GPU benchmark v1 +# Py-Boost GPU benchmarks -`pyboost-gpu-v1` is a fixed cross-version benchmark, not a collection of ad hoc +`pyboost-gpu-v1-k5` is the default fixed cross-version benchmark, not a collection of ad hoc experiments. Run this harness against clean Git checkouts using the same Python environment and one otherwise idle NVIDIA GPU. A GitHub Actions GPU runner is not required. The tested library version is selected by `--source`, independently @@ -8,17 +8,17 @@ of the directory containing the harness. ## Frozen workload -Definitions are in `suite-v1.json`. All cases build **32 iterations**, use learning +Default definitions are in `suite-v1-k5.json`. All cases build **32 iterations**, use learning rate 0.1, seed 42, full row/column sampling, and `use_hess=True`. Defaults are depth -4 and 64 bins. Every sketch case explicitly sets **`sketch_outputs=8`**. -Here **K** is the original number of outputs and **k=8** is the sketch width used +4 and 64 bins. Every sketch case explicitly sets **`sketch_outputs=5`**. +Here **K** is the original number of outputs and **k=5** is the sketch width used only for structure search; final predictions still contain all K outputs: -- `proj`: eight projected components combining the original outputs. -- `rand`: eight output draws with replacement, so at most eight distinct outputs. -- `filter`: eight importance-weighted output draws with replacement, so at most - eight distinct outputs. -- `topk`: eight distinct outputs with the largest gradient norms. +- `proj`: five projected components combining the original outputs. +- `rand`: five output draws with replacement, so at most five distinct outputs. +- `filter`: five importance-weighted output draws with replacement, so at most + five distinct outputs. +- `topk`: five distinct outputs with the largest gradient norms. - Full `GradientBoosting`: all K outputs, without a sketch. Early stopping is disabled. The benchmark counts executed iterations and @@ -30,16 +30,16 @@ with `es=0`. Different retained tree counts are disclosed in comparisons. |---|---:|---:|---:|---:|---| | mse-single | 16384 / 4096 / 4096 | 32 | 1 | all | GradientBoosting | | mse-full-128 | 8192 / 2048 / 2048 | 32 | 128 | all | GradientBoosting | -| mse-proj-32 | 16384 / 4096 / 4096 | 32 | 32 | 8 | SketchBoost/proj | -| mse-proj-128 | 16384 / 4096 / 4096 | 32 | 128 | 8 | SketchBoost/proj | -| mse-proj-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 8 | SketchBoost/proj | -| mse-proj-10000 | 16384 / 4096 / 4096 | 32 | 10000 | 8 | SketchBoost/proj | -| bce-rand-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 8 | SketchBoost/rand | -| bce-filter-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 8 | SketchBoost/filter | -| mse-topk-10000 | 4096 / 1024 / 1024 | 32 | 10000 | 8 | SketchBoost/topk | -| ce-proj-512 | 16384 / 4096 / 4096 | 32 | 512 classes | 8 | SketchBoost/proj | -| validation-proj-1024 | 8192 / 32768 / 4096 | 32 | 1024 | 8 | SketchBoost/proj; two validation sets | -| histogram-topk-128 | 8192 / 2048 / 2048 | 128 | 128 | 8 | SketchBoost/topk; depth 8, 256 bins | +| mse-proj-32 | 16384 / 4096 / 4096 | 32 | 32 | 5 | SketchBoost/proj | +| mse-proj-128 | 16384 / 4096 / 4096 | 32 | 128 | 5 | SketchBoost/proj | +| mse-proj-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 5 | SketchBoost/proj | +| mse-proj-10000 | 16384 / 4096 / 4096 | 32 | 10000 | 5 | SketchBoost/proj | +| bce-rand-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 5 | SketchBoost/rand | +| bce-filter-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 5 | SketchBoost/filter | +| mse-topk-10000 | 4096 / 1024 / 1024 | 32 | 10000 | 5 | SketchBoost/topk | +| ce-proj-512 | 16384 / 4096 / 4096 | 32 | 512 classes | 5 | SketchBoost/proj | +| validation-proj-1024 | 8192 / 32768 / 4096 | 32 | 1024 | 5 | SketchBoost/proj; two validation sets | +| histogram-topk-128 | 8192 / 2048 / 2048 | 128 | 128 | 5 | SketchBoost/topk; depth 8, 256 bins | The four `mse-proj-*` cases change only the number of outputs. Other cases exercise full-output scoring, separate/shared Hessians, wide targets, validation memory, @@ -73,15 +73,15 @@ git worktree add --detach ../Py-Boost-v0.5.2 v0.5.2 --device 1 --output .benchmark-results/v052-vs-current ``` -For the same 12 workloads with `sketch_outputs=5`, select the separate frozen -`pyboost-gpu-v1-k5` profile. The k=8 manifest remains unchanged; reports with -different sketch widths are not comparable: +The command above defaults to `sketch_outputs=5`. To reproduce the original +`pyboost-gpu-v1` workload with `sketch_outputs=8`, explicitly select its unchanged +manifest. Reports with different sketch widths are not comparable: ```bash .venv-benchmark/bin/python benchmarks/run.py run \ - --suite benchmarks/suite-v1-k5.json \ + --suite benchmarks/suite-v1.json \ --source /path/to/baseline --source /path/to/candidate \ - --device 1 --output .benchmark-results/k5-comparison + --device 1 --output .benchmark-results/k8-comparison ``` Commit changes to `py_boost/` or `pyproject.toml` first: dirty library sources are @@ -160,7 +160,7 @@ environment. Different GPUs/environments need separate baselines. Repeated runs can still differ due to temperature, clocks and scheduling: retain raw samples and investigate noise before calling a small difference a regression. -Treat v1's data, cases and measurement semantics as frozen. New semantics/workloads +Treat both profiles' data, cases and measurement semantics as frozen. New semantics/workloads need a new suite version. Any harness change alters its checksum and requires remeasuring both versions; never relabel old results to bypass that check. diff --git a/benchmarks/protocol.py b/benchmarks/protocol.py index 6c3ae44..99c7cf2 100644 --- a/benchmarks/protocol.py +++ b/benchmarks/protocol.py @@ -7,6 +7,7 @@ import numpy as np ROOT = Path(__file__).resolve().parent +DEFAULT_SUITE = ROOT / 'suite-v1-k5.json' SCHEMA_VERSION = 1 @@ -24,7 +25,7 @@ def implementation_hash(suite=None): def load_suite(path=None): - return json.loads((Path(path) if path is not None else ROOT / 'suite-v1.json').read_text()) + return json.loads((Path(path) if path is not None else DEFAULT_SUITE).read_text()) def make_data(case, seed): diff --git a/benchmarks/run.py b/benchmarks/run.py index e31103f..ec6f059 100644 --- a/benchmarks/run.py +++ b/benchmarks/run.py @@ -17,11 +17,11 @@ import traceback if __package__: - from .protocol import (ROOT, SCHEMA_VERSION, comparison_errors, digest_json, + from .protocol import (DEFAULT_SUITE, ROOT, SCHEMA_VERSION, comparison_errors, digest_json, implementation_hash, load_suite, make_data, quality, summarize) from .gpu import RssPeak, memory_tracker else: - from protocol import (ROOT, SCHEMA_VERSION, comparison_errors, digest_json, + from protocol import (DEFAULT_SUITE, ROOT, SCHEMA_VERSION, comparison_errors, digest_json, implementation_hash, load_suite, make_data, quality, summarize) from gpu import RssPeak, memory_tracker @@ -379,8 +379,8 @@ def main(): commands = parser.add_subparsers(dest='command', required=True) command = commands.add_parser('run', help='Run one or more clean source checkouts') command.add_argument('--source', action='append', required=True, help='Git checkout; repeat to compare versions') - command.add_argument('--suite', type=Path, default=ROOT / 'suite-v1.json', - help='Frozen suite manifest (default: suite-v1.json, k=8)') + command.add_argument('--suite', type=Path, default=DEFAULT_SUITE, + help='Frozen suite manifest (default: suite-v1-k5.json, k=5)') command.add_argument('--output', required=True) command.add_argument('--device', required=True, help='One physical GPU index or UUID') command.add_argument('--python', default=sys.executable, help='Interpreter containing the pinned dependencies') diff --git a/tests/test_benchmark_protocol.py b/tests/test_benchmark_protocol.py index bab357c..7c20f73 100644 --- a/tests/test_benchmark_protocol.py +++ b/tests/test_benchmark_protocol.py @@ -57,8 +57,9 @@ def test_same_multilabel_data_for_different_sketch_methods(): def test_k5_profile_changes_only_sketch_width_and_identity(): from copy import deepcopy - original = load_suite() + original = load_suite(ROOT / 'suite-v1.json') k5 = load_suite(ROOT / 'suite-v1-k5.json') + assert load_suite() == k5 assert k5['benchmark_id'] == 'pyboost-gpu-v1-k5' expected = deepcopy(original) expected['benchmark_id'] = k5['benchmark_id'] @@ -75,6 +76,20 @@ def test_k5_profile_changes_only_sketch_width_and_identity(): assert make_data(before, 42)[-1] == make_data(after, 42)[-1] +def test_cli_defaults_to_k5(monkeypatch, tmp_path): + captured = {} + def capture(args): + captured['suite'] = load_suite(args.suite) + return 0 + monkeypatch.setattr(runner, 'run', capture) + monkeypatch.setattr(sys, 'argv', ['run.py', 'run', '--source', 'checkout', + '--device', '1', '--output', str(tmp_path)]) + assert runner.main() == 0 + assert captured['suite']['benchmark_id'] == 'pyboost-gpu-v1-k5' + assert all(c['params']['sketch_outputs'] == 5 for c in captured['suite']['cases'] + if c['model'] == 'SketchBoost') + + def test_holdout_metrics_and_invalid_predictions(): assert quality('multilabel', np.array([[0, 1]], dtype=np.float32), np.array([[.5, .5]], dtype=np.float32))['value'] == pytest.approx(np.log(2)) From 6f190ed9c4bbc3fa79f1a7dfe156c88c7150ed8d Mon Sep 17 00:00:00 2001 From: Rinchin Date: Sat, 31 Oct 2026 01:08:41 +0300 Subject: [PATCH 7/9] Publish benchmark tables in PR comments and keep raw results local --- AGENTS.md | 8 ++++---- benchmarks/README.md | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d937d97..fb515e3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,10 +21,10 @@ with a single sketch method, a small-output case, or an improvised timing script - Use `benchmarks/run.py compare` to produce the report; do not compare mismatched environments/workloads. Inspect time, peak live CuPy bytes, CPU RSS, holdout quality, executed iterations and retained tree counts. -- When PR creation/update is authorized, publish the comparison table in the PR - and link both raw result sets from a permanent location, preferably the separate - `benchmark-results` branch. Publishing reports must not change the measured - library commit. For local-only work, provide paths to the saved reports. +- When PR creation/update is authorized, publish the comparison table and run + configuration in a PR comment. Keep raw JSON, logs and archives locally in the + ignored output directory. Do not create result branches or upload artifacts + unless explicitly requested. For local-only work, provide saved report paths. - After changing the library code, rerun against the new candidate SHA. An earlier report does not verify later code changes. For a change to the harness itself, rerun both library versions with the final harness implementation. diff --git a/benchmarks/README.md b/benchmarks/README.md index 09f4937..c3028aa 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -164,8 +164,8 @@ Treat both profiles' data, cases and measurement semantics as frozen. New semant need a new suite version. Any harness change alters its checksum and requires remeasuring both versions; never relabel old results to bypass that check. -Publish `comparison.md` in the PR and retain both `results.json` files and raw logs -in the repository's separate `benchmark-results` branch or another explicitly -agreed permanent location. Keep the measured code SHA unchanged when publishing -reports. The runner itself performs no network writes. Do not commit local results -or environment directories into the code branch. +Publish the comparison table and run configuration in a PR comment. Retain both +`results.json` files, raw logs and archives locally in the ignored output +directory. Do not create result branches or upload artifacts unless explicitly +requested. The runner itself performs no network writes. Do not commit local +results or environment directories into the code branch. From 4d2d89f67364472e93ce23b3658194c379ba123c Mon Sep 17 00:00:00 2001 From: Rinchin Date: Mon, 2 Nov 2026 11:49:26 +0300 Subject: [PATCH 8/9] Add Delicious paper protocol as the default local GPU benchmark --- .gitignore | 1 + AGENTS.md | 18 +- benchmarks/README.md | 307 ++++++------ benchmarks/SYNTHETIC.md | 172 +++++++ benchmarks/delicious.py | 105 ++++ benchmarks/protocol.py | 16 +- benchmarks/run.py | 120 ++++- benchmarks/suite-delicious-paper-v1-k5.json | 522 ++++++++++++++++++++ tests/test_benchmark_protocol.py | 5 +- tests/test_delicious_benchmark.py | 112 +++++ 10 files changed, 1189 insertions(+), 189 deletions(-) create mode 100644 benchmarks/SYNTHETIC.md create mode 100644 benchmarks/delicious.py create mode 100644 benchmarks/suite-delicious-paper-v1-k5.json create mode 100644 tests/test_delicious_benchmark.py diff --git a/.gitignore b/.gitignore index 0bd61ba..80f8b86 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ coverage.xml .hypothesis/ .pytest_cache/ .benchmark-results/ +.benchmark-data/ cover/ # Translations diff --git a/AGENTS.md b/AGENTS.md index fb515e3..9701a45 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,12 +4,16 @@ ## Required local GPU benchmark -Before declaring a PR ready, run the complete `pyboost-gpu-v1-k5` benchmark described -in `benchmarks/README.md` against the PR's exact base and candidate commits, using -the same harness, pinned environment and idle GPU. Use all 12 cases, one warmup -and five timing repetitions, with `sketch_outputs=5` in every sketch case; keep -memory runs separate. Do not replace the suite -with a single sketch method, a small-output case, or an improvised timing script. +Before declaring a PR ready, run the complete `pyboost-delicious-paper-v1-k5` +benchmark described in `benchmarks/README.md` against the PR's exact base and +candidate commits, using the same harness, pinned environment and idle GPU. +Use all 25 method/fold cases: five methods, five shuffled train folds, the official +held-out test set, all 983 labels and `sketch_outputs=5` in every sketch case. +Use the paper's parameters, up to 20000 iterations and early stopping after 500 +rounds. Each fold has one timing run after a 32-iteration warmup and a separate +full memory run. Five CV folds are not five timing repetitions. Do not substitute +synthetic data, truncate the iteration limit or drop methods/folds in a required +run. This protocol can take hours; there is no ten-minute limit. - Commit library changes first. Record both code SHAs and the benchmark/data checksums. Verify that the imported library comes from the intended checkout. @@ -33,5 +37,5 @@ with a single sketch method, a small-output case, or an improvised timing script such a PR as benchmark-verified. Request an explicit user waiver when needed. The frozen benchmark is for version-to-version comparisons. Keep its cases, -generation and measurement semantics stable; introduce a new suite version when +data, splits and measurement semantics stable; introduce a new suite version when these semantics change. GPU unit tests remain required and do not replace it. diff --git a/benchmarks/README.md b/benchmarks/README.md index c3028aa..3cfa830 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,171 +1,162 @@ -# Py-Boost GPU benchmarks - -`pyboost-gpu-v1-k5` is the default fixed cross-version benchmark, not a collection of ad hoc -experiments. Run this harness against clean Git checkouts using the same Python -environment and one otherwise idle NVIDIA GPU. A GitHub Actions GPU runner is -not required. The tested library version is selected by `--source`, independently -of the directory containing the harness. - -## Frozen workload - -Default definitions are in `suite-v1-k5.json`. All cases build **32 iterations**, use learning -rate 0.1, seed 42, full row/column sampling, and `use_hess=True`. Defaults are depth -4 and 64 bins. Every sketch case explicitly sets **`sketch_outputs=5`**. -Here **K** is the original number of outputs and **k=5** is the sketch width used -only for structure search; final predictions still contain all K outputs: - -- `proj`: five projected components combining the original outputs. -- `rand`: five output draws with replacement, so at most five distinct outputs. -- `filter`: five importance-weighted output draws with replacement, so at most - five distinct outputs. -- `topk`: five distinct outputs with the largest gradient norms. -- Full `GradientBoosting`: all K outputs, without a sketch. - -Early stopping is disabled. The benchmark counts executed iterations and -separately records retained -trees: existing Py-Boost versions can prune to the best validation iteration even -with `es=0`. Different retained tree counts are disclosed in comparisons. - -| Case | Train / validation / test rows | Features | Outputs K | Sketch width k | Model | -|---|---:|---:|---:|---:|---| -| mse-single | 16384 / 4096 / 4096 | 32 | 1 | all | GradientBoosting | -| mse-full-128 | 8192 / 2048 / 2048 | 32 | 128 | all | GradientBoosting | -| mse-proj-32 | 16384 / 4096 / 4096 | 32 | 32 | 5 | SketchBoost/proj | -| mse-proj-128 | 16384 / 4096 / 4096 | 32 | 128 | 5 | SketchBoost/proj | -| mse-proj-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 5 | SketchBoost/proj | -| mse-proj-10000 | 16384 / 4096 / 4096 | 32 | 10000 | 5 | SketchBoost/proj | -| bce-rand-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 5 | SketchBoost/rand | -| bce-filter-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 5 | SketchBoost/filter | -| mse-topk-10000 | 4096 / 1024 / 1024 | 32 | 10000 | 5 | SketchBoost/topk | -| ce-proj-512 | 16384 / 4096 / 4096 | 32 | 512 classes | 5 | SketchBoost/proj | -| validation-proj-1024 | 8192 / 32768 / 4096 | 32 | 1024 | 5 | SketchBoost/proj; two validation sets | -| histogram-topk-128 | 8192 / 2048 / 2048 | 128 | 128 | 5 | SketchBoost/topk; depth 8, 256 bins | - -The four `mse-proj-*` cases change only the number of outputs. Other cases exercise -full-output scoring, separate/shared Hessians, wide targets, validation memory, -and deep histograms. No sketch method is treated as the primary benchmark. -SVD requires an additional RAPIDS stack and is outside v1. Missing-label losses -are outside v1 to keep this suite runnable against released 0.5.2 as well as newer -versions; add such workloads under a new suite ID. - -Data are generated by `protocol.make_data` using PCG64 seed 20261030, float32 -features/targets and eight correlated output groups. Regression has additive -noise; multilabel targets are Bernoulli draws from signal-dependent probabilities. -Multiclass labels encode feature signs, with all classes represented in training. -Generation uses no matrix multiplication or external downloads. SHA-256 covers -the actual feature and target bytes. Validation/test rows are held out. The rand -and filter cases receive identical data. - -## Environment and commands - -The reference stack is Linux, Python 3.12 and CUDA 12. Install the complete pinned -dependency closure in a dedicated environment. This file deliberately does not -install a particular Py-Boost revision; install metadata once, then select each -tested checkout through `--source`. +# Delicious GPU benchmark + +The default `pyboost-delicious-paper-v1-k5` compares library revisions on the real +**Delicious** multilabel dataset from the +[SketchBoost paper, Appendix B](https://arxiv.org/html/2211.12858v1#A2). +It follows the paper's evaluation protocol and chosen hyperparameters, without +rerunning hyperparameter search or competing libraries. It can take hours on one +GPU; the old ten-minute synthetic budget no longer applies. + +## Frozen data and protocol + +Delicious has **16105 rows, 500 features and 983 binary labels**. Download the +original 1.3 MB archive selected by the +[paper's download script](https://github.com/sb-ai-lab/SketchBoost-paper/blob/e479fde7a0dd5f624fd9efd654e2d192a12d325d/get_data.sh), +not Delicious-200K. The harness checks the archive and every input file's SHA-256 +before parsing dense float32 arrays. The NPZ cache is loaded with +`allow_pickle=False`; no external pickle is loaded. Preparation is outside timing. + +Following the [reference preprocessing](https://github.com/sb-ai-lab/SketchBoost-paper/blob/e479fde7a0dd5f624fd9efd654e2d192a12d325d/preprocess.py), +use column zero of the official split files, preserving order and converting +one-based indices. There are **12920 train / 3185 test** rows. Following the +[reference runner](https://github.com/sb-ai-lab/SketchBoost-paper/blob/e479fde7a0dd5f624fd9efd654e2d192a12d325d/runner.py), +apply `KFold(5, shuffle=True, random_state=42)` only to train: every fold has +**10336 train / 2584 validation** rows. All five models are independently evaluated +on the untouched 3185-row test set. Do not average their predictions into an +ensemble. Model seed is 42; CPU thread count is eight. + +Archive SHA-256: +`9a4e79e9bd1835225e59eef09ba0ee29c5430f785045a9245943511836542806`. +Normalized float32 data and ordered split SHA-256: +`a53ca49cdce00c38f90225994ba01e6b352e9445d121df35fcd177a1cbd5c676`. +Every worker verifies the normalized cache and records a fold checksum covering +the actual dataset plus ordered train, validation and test indices. + +The frozen manifest is `suite-delicious-paper-v1-k5.json`: + +| Parameter | Value | +|---|---:| +| Loss / validation metric | binary cross-entropy over all labels and rows | +| Maximum iterations | 20000 | +| Learning rate | 0.015 | +| Early stopping patience | 500 | +| Maximum depth | 12 | +| Maximum bins | 256 | +| Minimum data in leaf | 76 | +| Row sampling | 0.623613094287341 | +| L2 leaf regularization | 13.057198025783617 | +| Column sampling | 1.0 | +| `use_hess` | false | + +The unrounded parameters are copied into plain JSON from the Delicious/CatBoost +entry of the paper's +[`runs/baselines_and_params.pkl`](https://github.com/sb-ai-lab/SketchBoost-paper/blob/e479fde7a0dd5f624fd9efd654e2d192a12d325d/runs/baselines_and_params.pkl). +Sharing CatBoost's parameters with SketchBoost and disabling Hessians in split +scoring follows the experiment code. The tested checkout supplies the actual +sketch implementations: this compares library revisions, rather than pinning the +2022 library. No hyperparameter search runs during PR checks. + +| Method | Outputs K | Sketch width k | Coverage | +|---|---:|---:|---| +| Full GradientBoosting, single output group | 983 | all 983 | Paper baseline: no sketch | +| SketchBoost/topk | 983 | 5 | Paper: Top Outputs | +| SketchBoost/rand | 983 | 5 | Paper: Random Sampling | +| SketchBoost/proj | 983 | 5 | Paper: Random Projections | +| SketchBoost/filter | 983 | 5 | Additional library coverage, outside the paper | + +There are **25 cases**: five methods × five folds. `sketch_outputs=5` affects only +structure search; leaves and predictions retain all 983 labels. `topk` chooses +five distinct labels; `rand`/`filter` make five draws with replacement (at most +five distinct labels); `proj` creates five components. The paper also sweeps other +k values; this required profile fixes k=5 as requested. + +## Run and resume + +Use Linux, Python 3.12 and CUDA 12 with the pinned dependency closure. The harness +is independent of tested checkouts, selected by `--source`. ```bash uv venv --python 3.12 .venv-benchmark uv pip install --python .venv-benchmark/bin/python -r benchmarks/requirements-py312-cu12.txt uv pip install --python .venv-benchmark/bin/python --no-deps -e . -git worktree add --detach ../Py-Boost-v0.5.2 v0.5.2 +.venv-benchmark/bin/python benchmarks/run.py prepare-data .venv-benchmark/bin/python benchmarks/run.py run \ - --source ../Py-Boost-v0.5.2 --source . \ - --device 1 --output .benchmark-results/v052-vs-current + --source /path/to/base --source /path/to/candidate \ + --device 1 --output .benchmark-results/delicious-comparison ``` -The command above defaults to `sketch_outputs=5`. To reproduce the original -`pyboost-gpu-v1` workload with `sketch_outputs=8`, explicitly select its unchanged -manifest. Reports with different sketch widths are not comparable: - -```bash -.venv-benchmark/bin/python benchmarks/run.py run \ - --suite benchmarks/suite-v1.json \ - --source /path/to/baseline --source /path/to/candidate \ - --device 1 --output .benchmark-results/k8-comparison -``` - -Commit changes to `py_boost/` or `pyproject.toml` first: dirty library sources are -rejected. **Exactly one physical GPU is used for the entire run.** Use a physical -GPU index or UUID (`nvidia-smi -L`). `--device 1` selects only physical GPU #1; it -is resolved to its physical UUID and exposed inside each worker as logical -`cuda:0` through `CUDA_VISIBLE_DEVICES`. UUID selection avoids differences between -CUDA and `nvidia-smi` ordinal ordering. -A comma-separated GPU list is rejected, and workers verify that exactly one CUDA -device is visible. Cases, versions and measurement phases run sequentially. -Do not share the selected GPU with another workload. Preflight performs a real allocation/kernel, rejects other -compute processes on the GPU, and records its UUID. CPU thread counts are fixed at -4. No GPU is a failure, not a skipped or CPU-emulated benchmark. - -Each source produces `//results.json` and raw worker -requests/results/logs. GPU count, model, physical and CUDA-visible VRAM, UUID, -PCI bus, compute capability, SM count, memory bus width, power limit, maximum SM -clock and NVIDIA/CUDA driver/runtime versions are recorded -in JSON and displayed in the comparison report. Each comparison row shows the -full output count K and sketch width k/method explicitly. -Two-source runs also produce `comparison.md`. More than two -sources may be measured in one run; compare any two results afterward: +For offline preparation, use `prepare-data --archive /path/to/Delicious.zip`. +A different archive is rejected. Pass `--data-dir /path/to/cache` to preparation +and run when using a nondefault cache. Workers never download data implicitly. + +Exactly **one physical GPU** runs both versions, all methods/folds and measurement +phases sequentially. Select an index or UUID; only that UUID is exposed as logical +CUDA device zero. Multiple visible GPUs or other compute processes on the GPU +are rejected. Reports include model, physical/CUDA-visible VRAM, UUID, PCI bus, +compute capability, SM count, memory bus width, power, maximum SM clock and +NVIDIA/CUDA driver/runtime versions. Use the same idle GPU and environment for +both versions. GitHub Actions does not need a GPU worker. + +Commit library changes before running. Reports record source SHAs, imported paths +and source checksums; cases alternate version order. Each method/fold is saved in +`//results.json` alongside raw worker jobs, results and logs. +Default timeout is **24 hours per worker**, including warmup, timing and memory. +Failed cases, OOM and timeouts are failures, never skipped passes. + +Repeat the same command with `--resume` to skip completed cases and retry unfinished +or failed cases. It refuses different source SHAs, suite, harness, environment, +case selection or repeat counts. Stop the original run before resuming. Without +`--resume`, output must be empty. Per-worker logs show progress every 500 iterations. + +`--case delicious-proj-fold0` is a diagnostic subset and never fulfills the full +requirement. `--repeats` changes repetitions **within each fold**; the default is +**one**, not five. All five CV folds are required. + +## Measurement and publication + +Each version/method/fold runs in a fresh subprocess. Warmup uses 32 iterations on +that fold and is excluded from timing. It does not run 20000 warmup iterations. + +1. Fit one fresh model with up to 20000 iterations and validation early stopping. + Time fit and test prediction separately using CUDA synchronization and + `perf_counter`. Fit includes quantization, transfers and validation. +2. Score test predictions outside timers: binary cross-entropy averaged over + labels/rows, clipping probabilities to `[1e-7, 1-1e-7]` as in the paper. Use + bounded float64 accumulation. Test data never control stopping. +3. Fit and predict with another fresh model for memory measurement. Hooks and RSS + sampling are disabled during timing. Record peak live rounded CuPy allocations + (including Py-Boost's private fit pool) and CPU RSS sampled every 5 ms. GPU peaks + exclude context, free pool cache and external allocations; they are not total + process VRAM. CPU RSS includes resident input data and can miss short peaks. + +The summary reports **five-fold means ± population standard deviations** for fit, +prediction and test log loss, plus maximum fit GPU allocation across folds. JSON +preserves each fold's timing/memory, test quality, executed iterations and retained +trees for both phases. Folds are different datasets, not five repeats of one fit. +The per-fold table and iteration/tree counts follow the summary. Faster convergence +must be assessed together with work performed and quality. + +Two-source runs generate `comparison.md`. Saved reports can be compared with: ```bash .venv-benchmark/bin/python benchmarks/run.py compare \ - .benchmark-results/older/COMMIT/results.json \ - .benchmark-results/newer/COMMIT/results.json \ - --output .benchmark-results/comparison.md + .benchmark-results/run/BASE_SHA/results.json \ + .benchmark-results/run/CANDIDATE_SHA/results.json \ + --output .benchmark-results/run/comparison.md ``` -`--python /path/to/environment/bin/python` selects the worker interpreter. -`--case mse-single --repeats 1` is a diagnostic subset, not the mandatory full -profile. Output directories must be empty so previous evidence is not overwritten. -Workers have a 900-second timeout including setup/warmup and both phases; failures and OOM retain -logs/results, and the command exits nonzero. A version lacking a required API is -reported as failed, not silently replaced by another implementation. - -## Measurement contract - -- Each case/version uses a fresh subprocess. When multiple - versions are measured, their order alternates by case. -- Each subprocess generates and hashes data, then warms one complete fit/predict. - Data generation, imports, JIT warmup and estimator construction are excluded - from timing. Quantization, CPU/GPU transfers and evaluation inside `fit()` are - included. Prediction receives CPU input and includes transfers back to CPU. -- Timing uses five fresh-model repetitions, `perf_counter` and device-wide CUDA - synchronization. Raw samples, median and p10/p90 are retained. These are warmed - timings, not cold-start latency. Holdout scoring is outside the timers. -- After the timing repetitions, memory uses a separate fresh-model fit/predict in - the same subprocess. Hooks/RSS sampling are disabled during timing. Keeping - both phases in one process avoids repeating Numba import-time compilation. - CuPy malloc/free hooks track **peak live rounded allocation bytes**, including - Py-Boost's private fit pool. This excludes free pool cache, CUDA context and - allocations outside CuPy's hooked pools; it is not total process VRAM. - Prediction tracking includes still-live allocations observed during fit. -- CPU RSS is sampled every 5 ms during fit/predict and includes the resident input - dataset. It is an approximation and may miss very short peaks. Absolute peaks - and the fit baseline are stored. -- Holdout RMSE or log loss is computed in bounded float64 blocks. Every prediction - must be finite and have the expected shape. Metric samples, actual iteration - counts and retained tree counts are stored, so faster runs can be assessed - together with model quality and work performed. - -## Cross-version comparability and publication - -Reports identify the exact library commit and library-source checksum, benchmark -implementation checksum, manifest checksum, actual dataset checksum, GPU UUID, -CPU/OS, Python, relevant dependency versions, CUDA driver/runtime, repetitions and -warmups. `compare` refuses mismatched workloads, datasets, environments or failed -measurements (exit code 2). Partial runs are marked `full_suite: false`; only a full -five-repeat run satisfies the PR requirement. Comparison tables show measured -changes, not an automatic statistical pass/fail verdict. - -To compare historical versions, run them through **this same harness** and pinned -environment. Different GPUs/environments need separate baselines. Repeated runs -can still differ due to temperature, clocks and scheduling: retain raw samples -and investigate noise before calling a small difference a regression. - -Treat both profiles' data, cases and measurement semantics as frozen. New semantics/workloads -need a new suite version. Any harness change alters its checksum and requires -remeasuring both versions; never relabel old results to bypass that check. - -Publish the comparison table and run configuration in a PR comment. Retain both -`results.json` files, raw logs and archives locally in the ignored output -directory. Do not create result branches or upload artifacts unless explicitly -requested. The runner itself performs no network writes. Do not commit local -results or environment directories into the code branch. +Comparison rejects mismatched workloads, datasets, environments and failed runs. +Partial runs are marked `full_suite: false`. Measurements are not automatic +statistical verdicts or a claim of exactly matching the paper's scores/timings on +different hardware/software. Repeated runs can vary with GPU clocks and temperature. + +Publish the table and run configuration in a PR comment. Keep raw JSON, logs, +archives and cached data locally in ignored directories. Do not create result +branches or upload artifacts unless explicitly requested. The runner performs no +network writes; only `prepare-data` downloads the pinned archive. + +Legacy synthetic workloads remain selectable with `--suite +benchmarks/suite-v1-k5.json` or `--suite benchmarks/suite-v1.json` (k=8); see +[SYNTHETIC.md](SYNTHETIC.md). They do not fulfill the Delicious requirement. Frozen +workloads, data, splits or measurement changes require a new suite ID and rerunning +both library versions with the same final harness. diff --git a/benchmarks/SYNTHETIC.md b/benchmarks/SYNTHETIC.md new file mode 100644 index 0000000..c74a75d --- /dev/null +++ b/benchmarks/SYNTHETIC.md @@ -0,0 +1,172 @@ +# Py-Boost GPU benchmarks + +`pyboost-gpu-v1-k5` is the legacy synthetic cross-version benchmark, not a collection of ad hoc +experiments. Run this harness against clean Git checkouts using the same Python +environment and one otherwise idle NVIDIA GPU. A GitHub Actions GPU runner is +not required. The tested library version is selected by `--source`, independently +of the directory containing the harness. + +## Frozen workload + +Definitions are in `suite-v1-k5.json`. All cases build **32 iterations**, use learning +rate 0.1, seed 42, full row/column sampling, and `use_hess=True`. Defaults are depth +4 and 64 bins. Every sketch case explicitly sets **`sketch_outputs=5`**. +Here **K** is the original number of outputs and **k=5** is the sketch width used +only for structure search; final predictions still contain all K outputs: + +- `proj`: five projected components combining the original outputs. +- `rand`: five output draws with replacement, so at most five distinct outputs. +- `filter`: five importance-weighted output draws with replacement, so at most + five distinct outputs. +- `topk`: five distinct outputs with the largest gradient norms. +- Full `GradientBoosting`: all K outputs, without a sketch. + +Early stopping is disabled. The benchmark counts executed iterations and +separately records retained +trees: existing Py-Boost versions can prune to the best validation iteration even +with `es=0`. Different retained tree counts are disclosed in comparisons. + +| Case | Train / validation / test rows | Features | Outputs K | Sketch width k | Model | +|---|---:|---:|---:|---:|---| +| mse-single | 16384 / 4096 / 4096 | 32 | 1 | all | GradientBoosting | +| mse-full-128 | 8192 / 2048 / 2048 | 32 | 128 | all | GradientBoosting | +| mse-proj-32 | 16384 / 4096 / 4096 | 32 | 32 | 5 | SketchBoost/proj | +| mse-proj-128 | 16384 / 4096 / 4096 | 32 | 128 | 5 | SketchBoost/proj | +| mse-proj-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 5 | SketchBoost/proj | +| mse-proj-10000 | 16384 / 4096 / 4096 | 32 | 10000 | 5 | SketchBoost/proj | +| bce-rand-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 5 | SketchBoost/rand | +| bce-filter-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 5 | SketchBoost/filter | +| mse-topk-10000 | 4096 / 1024 / 1024 | 32 | 10000 | 5 | SketchBoost/topk | +| ce-proj-512 | 16384 / 4096 / 4096 | 32 | 512 classes | 5 | SketchBoost/proj | +| validation-proj-1024 | 8192 / 32768 / 4096 | 32 | 1024 | 5 | SketchBoost/proj; two validation sets | +| histogram-topk-128 | 8192 / 2048 / 2048 | 128 | 128 | 5 | SketchBoost/topk; depth 8, 256 bins | + +The four `mse-proj-*` cases change only the number of outputs. Other cases exercise +full-output scoring, separate/shared Hessians, wide targets, validation memory, +and deep histograms. No sketch method is treated as the primary benchmark. +SVD requires an additional RAPIDS stack and is outside v1. Missing-label losses +are outside v1 to keep this suite runnable against released 0.5.2 as well as newer +versions; add such workloads under a new suite ID. + +Data are generated by `protocol.make_data` using PCG64 seed 20261030, float32 +features/targets and eight correlated output groups. Regression has additive +noise; multilabel targets are Bernoulli draws from signal-dependent probabilities. +Multiclass labels encode feature signs, with all classes represented in training. +Generation uses no matrix multiplication or external downloads. SHA-256 covers +the actual feature and target bytes. Validation/test rows are held out. The rand +and filter cases receive identical data. + +## Environment and commands + +The reference stack is Linux, Python 3.12 and CUDA 12. Install the complete pinned +dependency closure in a dedicated environment. This file deliberately does not +install a particular Py-Boost revision; install metadata once, then select each +tested checkout through `--source`. + +```bash +uv venv --python 3.12 .venv-benchmark +uv pip install --python .venv-benchmark/bin/python -r benchmarks/requirements-py312-cu12.txt +uv pip install --python .venv-benchmark/bin/python --no-deps -e . +git worktree add --detach ../Py-Boost-v0.5.2 v0.5.2 +.venv-benchmark/bin/python benchmarks/run.py run \ + --suite benchmarks/suite-v1-k5.json \ + --source ../Py-Boost-v0.5.2 --source . \ + --device 1 --output .benchmark-results/v052-vs-current +``` + +The command above explicitly selects synthetic data with `sketch_outputs=5`. To reproduce the original +`pyboost-gpu-v1` workload with `sketch_outputs=8`, explicitly select its unchanged +manifest. Reports with different sketch widths are not comparable: + +```bash +.venv-benchmark/bin/python benchmarks/run.py run \ + --suite benchmarks/suite-v1.json \ + --source /path/to/baseline --source /path/to/candidate \ + --device 1 --output .benchmark-results/k8-comparison +``` + +Commit changes to `py_boost/` or `pyproject.toml` first: dirty library sources are +rejected. **Exactly one physical GPU is used for the entire run.** Use a physical +GPU index or UUID (`nvidia-smi -L`). `--device 1` selects only physical GPU #1; it +is resolved to its physical UUID and exposed inside each worker as logical +`cuda:0` through `CUDA_VISIBLE_DEVICES`. UUID selection avoids differences between +CUDA and `nvidia-smi` ordinal ordering. +A comma-separated GPU list is rejected, and workers verify that exactly one CUDA +device is visible. Cases, versions and measurement phases run sequentially. +Do not share the selected GPU with another workload. Preflight performs a real allocation/kernel, rejects other +compute processes on the GPU, and records its UUID. CPU thread counts are fixed at +4. No GPU is a failure, not a skipped or CPU-emulated benchmark. + +Each source produces `//results.json` and raw worker +requests/results/logs. GPU count, model, physical and CUDA-visible VRAM, UUID, +PCI bus, compute capability, SM count, memory bus width, power limit, maximum SM +clock and NVIDIA/CUDA driver/runtime versions are recorded +in JSON and displayed in the comparison report. Each comparison row shows the +full output count K and sketch width k/method explicitly. +Two-source runs also produce `comparison.md`. More than two +sources may be measured in one run; compare any two results afterward: + +```bash +.venv-benchmark/bin/python benchmarks/run.py compare \ + .benchmark-results/older/COMMIT/results.json \ + .benchmark-results/newer/COMMIT/results.json \ + --output .benchmark-results/comparison.md +``` + +`--python /path/to/environment/bin/python` selects the worker interpreter. +`--case mse-single --repeats 1` is a diagnostic subset, not the mandatory full +profile. Output directories must be empty so previous evidence is not overwritten. +Workers have a 900-second timeout including setup/warmup and both phases; failures and OOM retain +logs/results, and the command exits nonzero. A version lacking a required API is +reported as failed, not silently replaced by another implementation. + +## Measurement contract + +- Each case/version uses a fresh subprocess. When multiple + versions are measured, their order alternates by case. +- Each subprocess generates and hashes data, then warms one complete fit/predict. + Data generation, imports, JIT warmup and estimator construction are excluded + from timing. Quantization, CPU/GPU transfers and evaluation inside `fit()` are + included. Prediction receives CPU input and includes transfers back to CPU. +- Timing uses five fresh-model repetitions, `perf_counter` and device-wide CUDA + synchronization. Raw samples, median and p10/p90 are retained. These are warmed + timings, not cold-start latency. Holdout scoring is outside the timers. +- After the timing repetitions, memory uses a separate fresh-model fit/predict in + the same subprocess. Hooks/RSS sampling are disabled during timing. Keeping + both phases in one process avoids repeating Numba import-time compilation. + CuPy malloc/free hooks track **peak live rounded allocation bytes**, including + Py-Boost's private fit pool. This excludes free pool cache, CUDA context and + allocations outside CuPy's hooked pools; it is not total process VRAM. + Prediction tracking includes still-live allocations observed during fit. +- CPU RSS is sampled every 5 ms during fit/predict and includes the resident input + dataset. It is an approximation and may miss very short peaks. Absolute peaks + and the fit baseline are stored. +- Holdout RMSE or log loss is computed in bounded float64 blocks. Every prediction + must be finite and have the expected shape. Metric samples, actual iteration + counts and retained tree counts are stored, so faster runs can be assessed + together with model quality and work performed. + +## Cross-version comparability and publication + +Reports identify the exact library commit and library-source checksum, benchmark +implementation checksum, manifest checksum, actual dataset checksum, GPU UUID, +CPU/OS, Python, relevant dependency versions, CUDA driver/runtime, repetitions and +warmups. `compare` refuses mismatched workloads, datasets, environments or failed +measurements (exit code 2). Partial runs are marked `full_suite: false`; only a full +five-repeat run satisfies the PR requirement. Comparison tables show measured +changes, not an automatic statistical pass/fail verdict. + +To compare historical versions, run them through **this same harness** and pinned +environment. Different GPUs/environments need separate baselines. Repeated runs +can still differ due to temperature, clocks and scheduling: retain raw samples +and investigate noise before calling a small difference a regression. + +Treat both profiles' data, cases and measurement semantics as frozen. New semantics/workloads +need a new suite version. Any harness change alters its checksum and requires +remeasuring both versions; never relabel old results to bypass that check. + +Publish the comparison table and run configuration in a PR comment. Retain both +`results.json` files, raw logs and archives locally in the ignored output +directory. Do not create result branches or upload artifacts unless explicitly +requested. The runner itself performs no network writes. Do not commit local +results or environment directories into the code branch. diff --git a/benchmarks/delicious.py b/benchmarks/delicious.py new file mode 100644 index 0000000..4b32957 --- /dev/null +++ b/benchmarks/delicious.py @@ -0,0 +1,105 @@ +"""Pinned Delicious data and the five-fold evaluation protocol from SketchBoost. + +No pickled data or models are downloaded or deserialized by this harness. +""" + +import hashlib +from pathlib import Path +import tempfile +from urllib.request import urlopen +from zipfile import ZipFile + +import numpy as np + +DATA_URL = 'https://drive.google.com/uc?export=download&id=0B3lPMIHmG6vGdG1jZ19VS2NWRVU' +ARCHIVE_SHA256 = '9a4e79e9bd1835225e59eef09ba0ee29c5430f785045a9245943511836542806' +FILES = { + 'Delicious/Delicious_data.txt': '7c6732b0ba3e2acb8b6f8cd147b83107f798e50fedf47c3429f549d53d971b9b', + 'Delicious/delicious_trSplit.txt': '8dbe2f693e62ae6e45ee617db8372fd5f547f4bb2f59e17c47f55c97b7977d1e', + 'Delicious/delicious_tstSplit.txt': 'aac14f9f84e6fe3f77d81db8ceb04be935e72adf22d9284d78d58d7763f242fa', +} +DEFAULT_DATA_DIR = Path(__file__).resolve().parents[1] / '.benchmark-data' + + +def array_hash(*arrays): + digest = hashlib.sha256() + for arr in arrays: + arr = np.ascontiguousarray(arr) + digest.update(str((arr.shape, arr.dtype.str)).encode()) + digest.update(memoryview(arr).cast('B')) + return digest.hexdigest() + + +def read_archive(archive): + if hashlib.sha256(Path(archive).read_bytes()).hexdigest() != ARCHIVE_SHA256: + raise ValueError('Delicious archive SHA-256 mismatch') + with ZipFile(archive) as zipped: + contents = {} + for name, expected in FILES.items(): + value = zipped.read(name) + if hashlib.sha256(value).hexdigest() != expected: + raise ValueError(f'Delicious file SHA-256 mismatch: {name}') + contents[name] = value.decode('ascii').splitlines() + lines = contents['Delicious/Delicious_data.txt'] + if lines[0].split() != ['16105', '500', '983'] or len(lines) != 16106: + raise ValueError('Unexpected Delicious dimensions') + X = np.zeros((16105, 500), dtype=' 16 * 1024 * 1024: + raise ValueError('Unexpected Delicious archive size') + output.write(block) + arrays = read_archive(archive) + prepared = Path(temporary) / 'delicious.npz' + np.savez_compressed(prepared, **dict(zip(['X', 'y', 'train', 'test'], arrays))) + prepared.replace(destination) + return destination, array_hash(*arrays) + + +def load_fold(case, seed, data_dir): + from sklearn.model_selection import KFold + + path = Path(data_dir or DEFAULT_DATA_DIR) / 'delicious.npz' + if not path.is_file(): + raise ValueError(f'Missing {path}; run benchmarks/run.py prepare-data first') + with np.load(path, allow_pickle=False) as cache: + X, y, train, test = [cache[name] for name in ['X', 'y', 'train', 'test']] + # Validate the actual normalized arrays every time, including split ordering. + if array_hash(X, y, train, test) != NORMALIZED_SHA256: + raise ValueError('Prepared Delicious dataset checksum mismatch; prepare it again') + folds = list(KFold(5, shuffle=True, random_state=seed).split(train)) + fit, valid = [train[indices] for indices in folds[case['fold']]] + if [len(fit), len(valid), len(test)] != [case['train'], case['valid'], case['test']]: + raise ValueError('Delicious fold sizes differ from manifest') + digest = array_hash(X, y, fit, valid, test) + return X[fit], y[fit], [{'X': X[valid], 'y': y[valid]}], X[test], y[test], digest + + +# Computed from the checked archive, float32 arrays and ordered official splits. +NORMALIZED_SHA256 = 'a53ca49cdce00c38f90225994ba01e6b352e9445d121df35fcd177a1cbd5c676' diff --git a/benchmarks/protocol.py b/benchmarks/protocol.py index 99c7cf2..bede891 100644 --- a/benchmarks/protocol.py +++ b/benchmarks/protocol.py @@ -7,7 +7,7 @@ import numpy as np ROOT = Path(__file__).resolve().parent -DEFAULT_SUITE = ROOT / 'suite-v1-k5.json' +DEFAULT_SUITE = ROOT / 'suite-delicious-paper-v1-k5.json' SCHEMA_VERSION = 1 @@ -18,7 +18,7 @@ def digest_json(value): def implementation_hash(suite=None): digest = hashlib.sha256() digest.update(digest_json(load_suite() if suite is None else suite).encode()) - for name in ['protocol.py', 'gpu.py', 'run.py']: + for name in ['protocol.py', 'gpu.py', 'run.py', 'delicious.py']: digest.update(name.encode()) digest.update((ROOT / name).read_bytes()) return digest.hexdigest() @@ -28,8 +28,14 @@ def load_suite(path=None): return json.loads((Path(path) if path is not None else DEFAULT_SUITE).read_text()) -def make_data(case, seed): +def make_data(case, seed, data_dir=None): """PCG64, float32 inputs, eight correlated output groups, no external data.""" + if case.get('dataset') == 'delicious': + if __package__: + from .delicious import load_fold + else: + from delicious import load_fold + return load_fold(case, seed, data_dir) rng = np.random.Generator(np.random.PCG64(seed)) n = case['train'] + case['valid'] + case['test'] X = rng.standard_normal((n, case['features']), dtype=np.float32) @@ -72,7 +78,7 @@ def make_data(case, seed): return X[:t], y[:t], validation, X[v:], y[v:], digest.hexdigest() -def quality(task, target, prediction): +def quality(task, target, prediction, clip=1e-15): """Holdout metric outside timed regions, with bounded float64 temporaries.""" if not np.isfinite(prediction).all(): raise ValueError('Non-finite predictions') @@ -93,7 +99,7 @@ def quality(task, target, prediction): else: if p.shape != y.shape: raise ValueError('Prediction output count differs from target') - p = np.clip(p, 1e-15, 1 - 1e-15) + p = np.clip(p, clip, 1 - clip) values = -(y * np.log(p) + (1 - y) * np.log1p(-p)) total += float(values.sum()) count += values.size diff --git a/benchmarks/run.py b/benchmarks/run.py index ec6f059..f5eb54e 100644 --- a/benchmarks/run.py +++ b/benchmarks/run.py @@ -118,6 +118,8 @@ def worker(job): import psutil logging.disable(logging.CRITICAL) + if job.get('implementation_hash', implementation_hash(job['suite'])) != implementation_hash(job['suite']): + raise RuntimeError('Benchmark harness changed after the run started') cp.cuda.Device(0).use() metadata = environment(cp, np, psutil, job['device']) if job['phase'] == 'metadata': @@ -130,7 +132,7 @@ def worker(job): if source_info(expected) != job['source']: raise RuntimeError('Library source changed after the run started') case, suite = job['case'], job['suite'] - X, y, valid, test_X, test_y, data_hash = make_data(case, suite['data_seed']) + X, y, valid, test_X, test_y, data_hash = make_data(case, suite['data_seed'], job.get('data_dir')) params = dict(suite['common_params'], **case['params']) params.update(loss=case['loss'], seed=suite['model_seed']) if case['model'] == 'SketchBoost': @@ -143,11 +145,13 @@ class Iterations(py_boost.Callback): def after_iteration(self, info): self.count += 1 + if self.count % 500 == 0: + print(f'{case["id"]}: {self.count} iterations', flush=True) return False - def new_model(): + def new_model(overrides=None): counter = Iterations() - model = getattr(py_boost, case['model'])(**params, callbacks=[counter]) + model = getattr(py_boost, case['model'])(**dict(params, **(overrides or {})), callbacks=[counter]) return model, counter def cleanup(model): @@ -160,7 +164,7 @@ def sync(): cp.cuda.Device(0).synchronize() for _ in range(suite['warmups']): - warm, _ = new_model() + warm, _ = new_model({'ntrees': suite['warmup_iterations']} if 'warmup_iterations' in suite else None) warm.fit(X, y, eval_sets=valid) warm.predict(test_X) sync() @@ -198,15 +202,21 @@ def measure(phase): 'fit_rss_peak_bytes': rss_fit.peak, 'predict_rss_peak_bytes': rss_predict.peak, 'fit_rss_baseline_bytes': rss_fit.baseline} - if counter.count != params['ntrees']: + if not 1 <= counter.count <= params['ntrees']: + raise RuntimeError(f'Invalid executed iteration count: {counter.count}') + if not params['es'] and counter.count != params['ntrees']: raise RuntimeError(f'Expected {params["ntrees"]} iterations, got {counter.count}') if prediction.shape != (case['test'], case['outputs']): raise RuntimeError(f'Unexpected prediction shape: {prediction.shape}') - score = quality(case['task'], test_y, prediction) + score = quality(case['task'], test_y, prediction, + clip=suite.get('dataset', {}).get('metric_clip', 1e-15)) if not np.isfinite(score['value']): raise RuntimeError('Non-finite holdout metric') measurements.append({'trained_iterations': counter.count, - 'retained_trees': len(model.models), 'quality': score}) + 'retained_trees': len(model.models), 'quality': score, + 'stopped_early': counter.count < params['ntrees']}) + print(f'{case["id"]} {phase}: iterations={counter.count}, retained={len(model.models)}, ' + f'test {score["name"]}={score["value"]:.8f}', flush=True) cleanup(model) del model, prediction result = {'status': 'ok', 'environment': metadata, 'data_hash': data_hash, @@ -264,6 +274,9 @@ def invoke(python, output, job, timeout): def run(args): suite = load_suite(args.suite) + harness_hash = implementation_hash(suite) + args.repeats = suite['repeats'] if args.repeats is None else args.repeats + args.timeout = suite.get('timeout', 900) if args.timeout is None else args.timeout sources = [source_info(p) for p in args.source] if len({s['commit'] for s in sources}) != len(sources): raise ValueError('Each source must have a different commit') @@ -274,30 +287,42 @@ def run(args): raise ValueError('Repeats and timeout must be positive') if ',' in args.device or not args.device: raise ValueError('Select exactly one GPU index or UUID') + if suite.get('dataset', {}).get('name') == 'Delicious': + if not (Path(args.data_dir) / 'delicious.npz').is_file(): + raise ValueError('Run benchmarks/run.py prepare-data before benchmarking Delicious') device = resolve_gpu(args.device) cases = [c for c in suite['cases'] if c['id'] in selected] output = Path(args.output).resolve() output.mkdir(parents=True, exist_ok=True) - if any(output.iterdir()): + if any(output.iterdir()) and not args.resume: raise ValueError('Output directory must be empty; preserve previous measurements') reports, directories = [], [] for source in sources: directory = output / source['commit'][:12] - (directory / 'raw').mkdir(parents=True) + (directory / 'raw').mkdir(parents=True, exist_ok=True) report = {'schema_version': SCHEMA_VERSION, 'benchmark_id': suite['benchmark_id'], - 'implementation_hash': implementation_hash(suite), 'suite_hash': digest_json(suite), + 'implementation_hash': harness_hash, 'suite_hash': digest_json(suite), 'started_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'source': source, 'case_ids': [c['id'] for c in cases], 'full_suite': len(cases) == len(suite['cases']), 'repeats': args.repeats, 'warmups': suite['warmups'], 'suite': suite, 'cases': {}, 'complete': False} - job = dict(source=source, phase='metadata', device=device, suite=suite) + job = dict(source=source, phase='metadata', device=device, suite=suite, + implementation_hash=harness_hash) probe = invoke(args.python, directory / 'raw' / 'metadata', job, args.timeout) if probe['status'] != 'ok': report['error'] = probe write_json(directory / 'results.json', report) raise RuntimeError(f'GPU/environment preflight failed; see {probe["log"]}') report['environment'] = probe['environment'] + if args.resume and (directory / 'results.json').is_file(): + previous = json.loads((directory / 'results.json').read_text()) + for key in ['source', 'implementation_hash', 'suite_hash', 'case_ids', + 'repeats', 'warmups', 'environment']: + if previous.get(key) != report.get(key): + raise ValueError(f'Cannot resume: different {key}') + report = previous + report['complete'] = False reports.append(report) directories.append(directory) write_json(directory / 'results.json', report) @@ -307,9 +332,14 @@ def run(args): if index % 2: order.reverse() for i in order: + if implementation_hash(suite) != harness_hash: + raise RuntimeError('Benchmark harness changed; preserve this run and start a new one') + if reports[i]['cases'].get(case['id'], {}).get('status') == 'ok': + continue print(f'{sources[i]["commit"][:12]} {case["id"]}', flush=True) job = dict(source=sources[i], phase='case', device=device, suite=suite, - case=case, repeats=args.repeats) + case=case, repeats=args.repeats, data_dir=str(Path(args.data_dir).resolve())) + job['implementation_hash'] = harness_hash record = invoke(args.python, directories[i] / 'raw' / case['id'], job, args.timeout) if record['status'] == 'ok' and record['environment'] != reports[i]['environment']: record.update(status='error', error='Environment changed during the run') @@ -344,12 +374,16 @@ def cuda_version(value): 'K is the full target dimension; k is the sketch width used for structure search.', '`proj`: k projected components; `rand`/`filter`: k draws with replacement (at most k distinct outputs);', '`topk`: k distinct outputs. Full GradientBoosting uses all K outputs. Predictions always have K outputs.', '', - 'Times are warmed wall-clock medians. GPU memory is peak live CuPy allocations,', + 'Times per case are warmed wall-clock medians. GPU memory is peak live CuPy allocations,', 'including the private fit pool; CUDA context, pool cache and external library allocations are excluded.', 'Quality is a holdout metric (lower is better). These are measured changes, not statistical verdicts.', '', '| Case | Outputs K | Sketch k | Fit s (base → PR) | Δ fit | Predict s (base → PR) | Δ predict | Fit MiB (base → PR) | Δ memory | Quality (base → PR) |', '|---|---:|---|---:|---:|---:|---:|---:|---:|---:|'] notes = [] + if base.get('suite', {}).get('dataset', {}).get('name') == 'Delicious': + # Cross-validation folds are different datasets, not timing repetitions. + table_start = next(i for i, line in enumerate(lines) if line.startswith('| Case |')) + lines[table_start:table_start] = cv_summary(base, head) for case_id in base['case_ids']: a, b = base['cases'][case_id], head['cases'][case_id] def pair(section, key, scale=1): @@ -371,22 +405,68 @@ def pair(section, key, scale=1): rb = [m['retained_trees'] for m in b['timing']['measurements']] if len(set(ra + rb)) > 1: notes.append(f'Retained tree counts differ for `{case_id}`: {ra} vs {rb}; interpret prediction timing with care.') + if 'fold' in spec: + ta = [m['trained_iterations'] for m in a['timing']['measurements']] + tb = [m['trained_iterations'] for m in b['timing']['measurements']] + notes.append(f'`{case_id}`: executed {ta} → {tb}; retained {ra} → {rb}.') return '\n'.join(lines + [''] + notes) + '\n' +def cv_summary(base, head): + import numpy as np + + lines = ['**Delicious: official first split; five shuffled train folds (seed 42).**', + 'All 983 labels and 500 features; each fold: 10336 train / 2584 validation / 3185 test.', + 'Five-fold means ± population standard deviations below; folds are not timing repeats.', + 'Each memory peak is from a separate full fit; report the maximum across folds.', + 'FilterSketch is an additional library coverage case, outside the paper.', '', + '| Method | K | k | Mean fit s (base → PR) | Δ fit | Mean predict s (base → PR) | Max fit MiB (base → PR) | Mean test log loss (base → PR) |', + '|---|---:|---:|---|---:|---|---|---|'] + groups = list(dict.fromkeys(base['cases'][c]['spec']['group'] for c in base['case_ids'])) + for group in groups: + ids = [c for c in base['case_ids'] if base['cases'][c]['spec']['group'] == group] + if len(ids) != 5: + lines.append(f'Incomplete CV group `{group}`: {len(ids)}/5 folds; no five-fold aggregate.') + continue + def values(report, section, key): + return [report['cases'][c][section][key] for c in ids] + def mean_pair(key): + pairs = [] + means = [] + for report in [base, head]: + data = [v['median'] for v in values(report, 'timing', key)] + means.append(float(np.mean(data))) + pairs.append(f'{means[-1]:.6g} ± {np.std(data):.3g}') + return ' → '.join(pairs), means + fit, means = mean_pair('fit_seconds') + predict, _ = mean_pair('predict_seconds') + score, _ = mean_pair('quality') + peaks = [max(values(r, 'memory', 'fit_cupy_peak_bytes')) / 2**20 for r in [base, head]] + spec = base['cases'][ids[0]]['spec'] + k = spec['params'].get('sketch_outputs', 'all') + lines.append(f'| {group} | 983 | {k} | {fit} | {(means[1]/means[0]-1)*100:+.1f}% | {predict} | ' + f'{peaks[0]:.2f} → {peaks[1]:.2f} | {score} |') + return lines + ['', 'Per-fold measurements:', ''] + + def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest='command', required=True) command = commands.add_parser('run', help='Run one or more clean source checkouts') command.add_argument('--source', action='append', required=True, help='Git checkout; repeat to compare versions') command.add_argument('--suite', type=Path, default=DEFAULT_SUITE, - help='Frozen suite manifest (default: suite-v1-k5.json, k=5)') + help='Frozen suite manifest (default: Delicious paper protocol, k=5)') command.add_argument('--output', required=True) command.add_argument('--device', required=True, help='One physical GPU index or UUID') command.add_argument('--python', default=sys.executable, help='Interpreter containing the pinned dependencies') - command.add_argument('--repeats', type=int, default=load_suite()['repeats']) + command.add_argument('--repeats', type=int, default=None, help='Timing repeats per fold; default from suite') command.add_argument('--case', action='append', help='Diagnostic subset; omit for the full suite') - command.add_argument('--timeout', type=int, default=900, help='Seconds per worker, including warmup') + command.add_argument('--timeout', type=int, default=None, help='Seconds per worker; default from suite') + command.add_argument('--resume', action='store_true', help='Resume matching reports, skipping successful cases') + command.add_argument('--data-dir', type=Path, default=ROOT.parent / '.benchmark-data') + command = commands.add_parser('prepare-data', help='Download and verify the pinned Delicious dataset') + command.add_argument('--data-dir', type=Path, default=ROOT.parent / '.benchmark-data') + command.add_argument('--archive', type=Path, help='Use an existing original Delicious.zip without network') command = commands.add_parser('compare', help='Compare two archived results.json files') command.add_argument('base') command.add_argument('candidate') @@ -395,6 +475,14 @@ def main(): command.add_argument('job') command.add_argument('result') args = parser.parse_args() + if args.command == 'prepare-data': + if __package__: + from .delicious import prepare_data + else: + from delicious import prepare_data + destination, checksum = prepare_data(args.data_dir, args.archive) + print(f'{destination}\nSHA-256 (arrays and ordered splits): {checksum}') + return 0 if args.command == '_worker': try: result = worker(json.loads(Path(args.job).read_text())) diff --git a/benchmarks/suite-delicious-paper-v1-k5.json b/benchmarks/suite-delicious-paper-v1-k5.json new file mode 100644 index 0000000..0a7e2a3 --- /dev/null +++ b/benchmarks/suite-delicious-paper-v1-k5.json @@ -0,0 +1,522 @@ +{ + "benchmark_id": "pyboost-delicious-paper-v1-k5", + "data_seed": 42, + "model_seed": 42, + "cpu_threads": 8, + "warmups": 1, + "repeats": 1, + "sketch_params": { + "proj": { + "norm": true + }, + "rand": { + "smooth": 0.1, + "replace": true + }, + "filter": { + "sample": true, + "smooth": 0.1, + "ntrees": 1 + }, + "topk": {} + }, + "common_params": { + "ntrees": 20000, + "max_depth": 12, + "max_bin": 256, + "lr": 0.015, + "es": 500, + "verbose": 1000000, + "min_data_in_leaf": 76, + "colsample": 1.0, + "subsample": 0.623613094287341, + "use_hess": false, + "lambda_l2": 13.057198025783617, + "gd_steps": 1, + "min_gain_to_split": 0, + "quantization": "Quantile", + "quant_sample": 2000000, + "min_data_in_bin": 3 + }, + "cases": [ + { + "id": "delicious-full-fold0", + "dataset": "delicious", + "fold": 0, + "group": "delicious-full", + "task": "multilabel", + "model": "GradientBoosting", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": {} + }, + { + "id": "delicious-full-fold1", + "dataset": "delicious", + "fold": 1, + "group": "delicious-full", + "task": "multilabel", + "model": "GradientBoosting", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": {} + }, + { + "id": "delicious-full-fold2", + "dataset": "delicious", + "fold": 2, + "group": "delicious-full", + "task": "multilabel", + "model": "GradientBoosting", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": {} + }, + { + "id": "delicious-full-fold3", + "dataset": "delicious", + "fold": 3, + "group": "delicious-full", + "task": "multilabel", + "model": "GradientBoosting", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": {} + }, + { + "id": "delicious-full-fold4", + "dataset": "delicious", + "fold": 4, + "group": "delicious-full", + "task": "multilabel", + "model": "GradientBoosting", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": {} + }, + { + "id": "delicious-topk-fold0", + "dataset": "delicious", + "fold": 0, + "group": "delicious-topk", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "topk", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-topk-fold1", + "dataset": "delicious", + "fold": 1, + "group": "delicious-topk", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "topk", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-topk-fold2", + "dataset": "delicious", + "fold": 2, + "group": "delicious-topk", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "topk", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-topk-fold3", + "dataset": "delicious", + "fold": 3, + "group": "delicious-topk", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "topk", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-topk-fold4", + "dataset": "delicious", + "fold": 4, + "group": "delicious-topk", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "topk", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-rand-fold0", + "dataset": "delicious", + "fold": 0, + "group": "delicious-rand", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "rand", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-rand-fold1", + "dataset": "delicious", + "fold": 1, + "group": "delicious-rand", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "rand", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-rand-fold2", + "dataset": "delicious", + "fold": 2, + "group": "delicious-rand", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "rand", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-rand-fold3", + "dataset": "delicious", + "fold": 3, + "group": "delicious-rand", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "rand", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-rand-fold4", + "dataset": "delicious", + "fold": 4, + "group": "delicious-rand", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "rand", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-proj-fold0", + "dataset": "delicious", + "fold": 0, + "group": "delicious-proj", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-proj-fold1", + "dataset": "delicious", + "fold": 1, + "group": "delicious-proj", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-proj-fold2", + "dataset": "delicious", + "fold": 2, + "group": "delicious-proj", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-proj-fold3", + "dataset": "delicious", + "fold": 3, + "group": "delicious-proj", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-proj-fold4", + "dataset": "delicious", + "fold": 4, + "group": "delicious-proj", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-filter-fold0", + "dataset": "delicious", + "fold": 0, + "group": "delicious-filter", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "filter", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-filter-fold1", + "dataset": "delicious", + "fold": 1, + "group": "delicious-filter", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "filter", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-filter-fold2", + "dataset": "delicious", + "fold": 2, + "group": "delicious-filter", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "filter", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-filter-fold3", + "dataset": "delicious", + "fold": 3, + "group": "delicious-filter", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "filter", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-filter-fold4", + "dataset": "delicious", + "fold": 4, + "group": "delicious-filter", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "filter", + "sketch_outputs": 5 + } + } + ], + "warmup_iterations": 32, + "timeout": 86400, + "dataset": { + "name": "Delicious", + "rows": 16105, + "features": 500, + "outputs": 983, + "archive_sha256": "9a4e79e9bd1835225e59eef09ba0ee29c5430f785045a9245943511836542806", + "normalized_sha256": "a53ca49cdce00c38f90225994ba01e6b352e9445d121df35fcd177a1cbd5c676", + "source": "https://arxiv.org/html/2211.12858v1", + "reference_commit": "e479fde7a0dd5f624fd9efd654e2d192a12d325d", + "official_split_column": 0, + "cv": { + "folds": 5, + "shuffle": true, + "random_state": 42 + }, + "metric_clip": 1e-07 + } +} diff --git a/tests/test_benchmark_protocol.py b/tests/test_benchmark_protocol.py index 7c20f73..bf7406a 100644 --- a/tests/test_benchmark_protocol.py +++ b/tests/test_benchmark_protocol.py @@ -47,7 +47,7 @@ def test_datasets_are_reproducible_and_separated(task): def test_same_multilabel_data_for_different_sketch_methods(): - suite = load_suite() + suite = load_suite(ROOT / 'suite-v1-k5.json') rand, filt = [next(c for c in suite['cases'] if c['id'] == name) for name in ['bce-rand-1024', 'bce-filter-1024']] for case in [rand, filt]: @@ -59,7 +59,6 @@ def test_k5_profile_changes_only_sketch_width_and_identity(): from copy import deepcopy original = load_suite(ROOT / 'suite-v1.json') k5 = load_suite(ROOT / 'suite-v1-k5.json') - assert load_suite() == k5 assert k5['benchmark_id'] == 'pyboost-gpu-v1-k5' expected = deepcopy(original) expected['benchmark_id'] = k5['benchmark_id'] @@ -85,7 +84,7 @@ def capture(args): monkeypatch.setattr(sys, 'argv', ['run.py', 'run', '--source', 'checkout', '--device', '1', '--output', str(tmp_path)]) assert runner.main() == 0 - assert captured['suite']['benchmark_id'] == 'pyboost-gpu-v1-k5' + assert captured['suite']['benchmark_id'] == 'pyboost-delicious-paper-v1-k5' assert all(c['params']['sketch_outputs'] == 5 for c in captured['suite']['cases'] if c['model'] == 'SketchBoost') diff --git a/tests/test_delicious_benchmark.py b/tests/test_delicious_benchmark.py new file mode 100644 index 0000000..be8196f --- /dev/null +++ b/tests/test_delicious_benchmark.py @@ -0,0 +1,112 @@ +"""Dataset integrity, official split isolation and five-fold report aggregation.""" + +from copy import deepcopy +import json +from types import SimpleNamespace + +import numpy as np +import pytest + +from test_benchmark_protocol import report +from pyboost_benchmark_suite import delicious +from pyboost_benchmark_suite.protocol import load_suite +from pyboost_benchmark_suite.run import cv_summary +from pyboost_benchmark_suite import run as runner + + +def test_paper_profile_has_all_folds_and_outputs(): + suite = load_suite() + assert suite['benchmark_id'] == 'pyboost-delicious-paper-v1-k5' + assert len(suite['cases']) == 25 + for method in ['full', 'topk', 'rand', 'proj', 'filter']: + cases = [c for c in suite['cases'] if c['group'] == f'delicious-{method}'] + assert {c['fold'] for c in cases} == set(range(5)) + assert all(c['outputs'] == 983 for c in cases) + params = suite['common_params'] + assert (params['ntrees'], params['es'], params['lr'], params['max_depth']) == (20000, 500, .015, 12) + assert params['use_hess'] is False + assert suite['repeats'] == 1 # Different folds, rather than five repeats per fold. + + +def test_corrupted_archive_rejected_before_parsing(tmp_path): + archive = tmp_path / 'Delicious.zip' + archive.write_bytes(b'not the official archive') + with pytest.raises(ValueError, match='archive SHA-256 mismatch'): + delicious.prepare_data(tmp_path, archive) + assert not (tmp_path / 'delicious.npz').exists() + + +def test_official_test_set_never_enters_folds_and_validation_covers_train(monkeypatch, tmp_path): + # Tiny checked arrays avoid downloading a dataset during unit tests. + X = np.arange(60, dtype=np.float32).reshape(20, 3) + y = np.zeros((20, 2), dtype=np.float32) + train, test = np.arange(15), np.arange(15, 20) + np.savez(tmp_path / 'delicious.npz', X=X, y=y, train=train, test=test) + monkeypatch.setattr(delicious, 'NORMALIZED_SHA256', delicious.array_hash(X, y, train, test)) + validations = [] + hashes = [] + for fold in range(5): + case = dict(fold=fold, train=12, valid=3, test=5) + a, b, valid, tx, ty, digest = delicious.load_fold(case, 42, tmp_path) + np.testing.assert_array_equal(tx, X[test]) + np.testing.assert_array_equal(ty, y[test]) + fit_ids = set(a[:, 0]) + valid_ids = set(valid[0]['X'][:, 0]) + assert fit_ids.isdisjoint(valid_ids) + assert (fit_ids | valid_ids) == set(X[train, 0]) + validations.extend(valid_ids) + hashes.append(digest) + assert digest == delicious.load_fold(case, 42, tmp_path)[-1] + assert sorted(validations) == sorted(X[train, 0]) + assert len(set(hashes)) == 5 + y[0, 0] = 1 + np.savez(tmp_path / 'delicious.npz', X=X, y=y, train=train, test=test) + with pytest.raises(ValueError, match='checksum mismatch'): + delicious.load_fold(case, 42, tmp_path) + + +def test_cv_uses_mean_not_median_and_maximum_memory(): + base = report() + sample = base['cases']['test'] + base['cases'] = {} + base['case_ids'] = [] + for fold, seconds in enumerate([1., 1., 1., 1., 6.]): + key = f'fold{fold}' + value = deepcopy(sample) + value['spec'] = dict(group='delicious-proj', fold=fold, params=dict(sketch_outputs=5)) + value['timing']['fit_seconds']['median'] = seconds + value['memory']['fit_cupy_peak_bytes'] = (fold + 1) * 2**20 + base['cases'][key] = value + base['case_ids'].append(key) + candidate = deepcopy(base) + text = '\n'.join(cv_summary(base, candidate)) + assert '| delicious-proj | 983 | 5 | 2 ± 2 → 2 ± 2 |' in text + assert '| 5.00 → 5.00 |' in text + base['case_ids'].pop() + assert 'Incomplete CV group' in '\n'.join(cv_summary(base, base)) + + +def test_resume_skips_completed_folds_and_rejects_changed_sources(monkeypatch, tmp_path): + suite = load_suite() + suite.pop('dataset') + suite['cases'] = suite['cases'][:1] + manifest = tmp_path / 'suite.json' + manifest.write_text(json.dumps(suite)) + calls = [] + source = dict(commit='123456789abc', path='checkout', library_hash='original') + monkeypatch.setattr(runner, 'source_info', lambda path: source.copy()) + monkeypatch.setattr(runner, 'resolve_gpu', lambda selector: 'GPU-test') + def invoke(python, path, job, timeout): + calls.append(job['phase']) + return dict(status='ok', environment={'gpu': 'test'}) + monkeypatch.setattr(runner, 'invoke', invoke) + args = SimpleNamespace(suite=manifest, source=['checkout'], repeats=None, + timeout=None, case=None, device='1', python='python', + output=tmp_path / 'out', data_dir=tmp_path, resume=False) + assert runner.run(args) == 0 + args.resume = True + assert runner.run(args) == 0 + assert calls == ['metadata', 'case', 'metadata'] + source['library_hash'] = 'changed' + with pytest.raises(ValueError, match='different source'): + runner.run(args) From d7be28b276144edaa97cee302922bff37e5971e0 Mon Sep 17 00:00:00 2001 From: Rinchin Date: Mon, 2 Nov 2026 12:11:40 +0300 Subject: [PATCH 9/9] Match paper sample deviations and allow long Delicious fits --- AGENTS.md | 2 +- benchmarks/README.md | 8 ++++---- benchmarks/SYNTHETIC.md | 5 +++-- benchmarks/run.py | 4 ++-- benchmarks/suite-delicious-paper-v1-k5.json | 2 +- tests/test_delicious_benchmark.py | 2 +- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9701a45..459d77a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ Use the paper's parameters, up to 20000 iterations and early stopping after 500 rounds. Each fold has one timing run after a 32-iteration warmup and a separate full memory run. Five CV folds are not five timing repetitions. Do not substitute synthetic data, truncate the iteration limit or drop methods/folds in a required -run. This protocol can take hours; there is no ten-minute limit. +run. This protocol can take days; there is no ten-minute limit. - Commit library changes first. Record both code SHAs and the benchmark/data checksums. Verify that the imported library comes from the intended checkout. diff --git a/benchmarks/README.md b/benchmarks/README.md index 3cfa830..578b704 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -4,8 +4,8 @@ The default `pyboost-delicious-paper-v1-k5` compares library revisions on the re **Delicious** multilabel dataset from the [SketchBoost paper, Appendix B](https://arxiv.org/html/2211.12858v1#A2). It follows the paper's evaluation protocol and chosen hyperparameters, without -rerunning hyperparameter search or competing libraries. It can take hours on one -GPU; the old ten-minute synthetic budget no longer applies. +rerunning hyperparameter search or competing libraries. The full comparison can take days on one GPU, especially the unsketched baseline +with separate memory fits; the old ten-minute synthetic budget no longer applies. ## Frozen data and protocol @@ -100,7 +100,7 @@ both versions. GitHub Actions does not need a GPU worker. Commit library changes before running. Reports record source SHAs, imported paths and source checksums; cases alternate version order. Each method/fold is saved in `//results.json` alongside raw worker jobs, results and logs. -Default timeout is **24 hours per worker**, including warmup, timing and memory. +Default timeout is **72 hours per worker**, including warmup, timing and memory. Failed cases, OOM and timeouts are failures, never skipped passes. Repeat the same command with `--resume` to skip completed cases and retry unfinished @@ -129,7 +129,7 @@ that fold and is excluded from timing. It does not run 20000 warmup iterations. exclude context, free pool cache and external allocations; they are not total process VRAM. CPU RSS includes resident input data and can miss short peaks. -The summary reports **five-fold means ± population standard deviations** for fit, +The summary reports **five-fold means ± sample standard deviations (`ddof=1`)** for fit, prediction and test log loss, plus maximum fit GPU allocation across folds. JSON preserves each fold's timing/memory, test quality, executed iterations and retained trees for both phases. Folds are different datasets, not five repeats of one fit. diff --git a/benchmarks/SYNTHETIC.md b/benchmarks/SYNTHETIC.md index c74a75d..fc93ce8 100644 --- a/benchmarks/SYNTHETIC.md +++ b/benchmarks/SYNTHETIC.md @@ -114,7 +114,7 @@ sources may be measured in one run; compare any two results afterward: ``` `--python /path/to/environment/bin/python` selects the worker interpreter. -`--case mse-single --repeats 1` is a diagnostic subset, not the mandatory full +`--case mse-single --repeats 1` is a diagnostic subset of this legacy profile. Output directories must be empty so previous evidence is not overwritten. Workers have a 900-second timeout including setup/warmup and both phases; failures and OOM retain logs/results, and the command exits nonzero. A version lacking a required API is @@ -153,7 +153,8 @@ implementation checksum, manifest checksum, actual dataset checksum, GPU UUID, CPU/OS, Python, relevant dependency versions, CUDA driver/runtime, repetitions and warmups. `compare` refuses mismatched workloads, datasets, environments or failed measurements (exit code 2). Partial runs are marked `full_suite: false`; only a full -five-repeat run satisfies the PR requirement. Comparison tables show measured +five-repeat run completes this legacy profile. The current PR requirement uses +Delicious; see README.md. Comparison tables show measured changes, not an automatic statistical pass/fail verdict. To compare historical versions, run them through **this same harness** and pinned diff --git a/benchmarks/run.py b/benchmarks/run.py index f5eb54e..50b67df 100644 --- a/benchmarks/run.py +++ b/benchmarks/run.py @@ -417,7 +417,7 @@ def cv_summary(base, head): lines = ['**Delicious: official first split; five shuffled train folds (seed 42).**', 'All 983 labels and 500 features; each fold: 10336 train / 2584 validation / 3185 test.', - 'Five-fold means ± population standard deviations below; folds are not timing repeats.', + 'Five-fold means ± sample standard deviations (ddof=1), as in the paper; folds are not timing repeats.', 'Each memory peak is from a separate full fit; report the maximum across folds.', 'FilterSketch is an additional library coverage case, outside the paper.', '', '| Method | K | k | Mean fit s (base → PR) | Δ fit | Mean predict s (base → PR) | Max fit MiB (base → PR) | Mean test log loss (base → PR) |', @@ -436,7 +436,7 @@ def mean_pair(key): for report in [base, head]: data = [v['median'] for v in values(report, 'timing', key)] means.append(float(np.mean(data))) - pairs.append(f'{means[-1]:.6g} ± {np.std(data):.3g}') + pairs.append(f'{means[-1]:.6g} ± {np.std(data, ddof=1):.3g}') return ' → '.join(pairs), means fit, means = mean_pair('fit_seconds') predict, _ = mean_pair('predict_seconds') diff --git a/benchmarks/suite-delicious-paper-v1-k5.json b/benchmarks/suite-delicious-paper-v1-k5.json index 0a7e2a3..4e361eb 100644 --- a/benchmarks/suite-delicious-paper-v1-k5.json +++ b/benchmarks/suite-delicious-paper-v1-k5.json @@ -501,7 +501,7 @@ } ], "warmup_iterations": 32, - "timeout": 86400, + "timeout": 259200, "dataset": { "name": "Delicious", "rows": 16105, diff --git a/tests/test_delicious_benchmark.py b/tests/test_delicious_benchmark.py index be8196f..17466fa 100644 --- a/tests/test_delicious_benchmark.py +++ b/tests/test_delicious_benchmark.py @@ -80,7 +80,7 @@ def test_cv_uses_mean_not_median_and_maximum_memory(): base['case_ids'].append(key) candidate = deepcopy(base) text = '\n'.join(cv_summary(base, candidate)) - assert '| delicious-proj | 983 | 5 | 2 ± 2 → 2 ± 2 |' in text + assert '| delicious-proj | 983 | 5 | 2 ± 2.24 → 2 ± 2.24 |' in text assert '| 5.00 → 5.00 |' in text base['case_ids'].pop() assert 'Incomplete CV group' in '\n'.join(cv_summary(base, base))