diff --git a/.gitignore b/.gitignore index 75677ef..80f8b86 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,8 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +.benchmark-results/ +.benchmark-data/ cover/ # Translations diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..459d77a --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,41 @@ +# Repository instructions + +уточняй непонятные вопросы перед изменением кода + +## Required local GPU benchmark + +Before declaring a PR ready, run the complete `pyboost-delicious-paper-v1-k5` +benchmark described in `benchmarks/README.md` against the PR's exact base and +candidate commits, using the same harness, pinned environment and idle GPU. +Use all 25 method/fold cases: five methods, five shuffled train folds, the official +held-out test set, all 983 labels and `sketch_outputs=5` in every sketch case. +Use the paper's parameters, up to 20000 iterations and early stopping after 500 +rounds. Each fold has one timing run after a 32-iteration warmup and a separate +full memory run. Five CV folds are not five timing repetitions. Do not substitute +synthetic data, truncate the iteration limit or drop methods/folds in a required +run. This protocol can take days; there is no ten-minute limit. + +- Commit library changes first. Record both code SHAs and the benchmark/data + checksums. Verify that the imported library comes from the intended checkout. +- Run `benchmarks/run.py run` with both source paths, an explicit device and a new + output directory. A missing GPU, failed case, timeout or OOM is not a pass. +- Use exactly one physical GPU. Never run benchmark cases or versions in parallel + on multiple GPUs. Publish its model, VRAM, UUID, compute capability and driver + versions in the report, plus full output count K and sketch width k per case. +- Use `benchmarks/run.py compare` to produce the report; do not compare mismatched + environments/workloads. Inspect time, peak live CuPy bytes, CPU RSS, holdout + quality, executed iterations and retained tree counts. +- When PR creation/update is authorized, publish the comparison table and run + configuration in a PR comment. Keep raw JSON, logs and archives locally in the + ignored output directory. Do not create result branches or upload artifacts + unless explicitly requested. For local-only work, provide saved report paths. +- After changing the library code, rerun against the new candidate SHA. An earlier + report does not verify later code changes. For a change to the harness itself, + rerun both library versions with the final harness implementation. +- Explicitly report regressions, failures, omitted cases and unavailable hardware. + Preserve completed work and evidence if a run cannot finish; do not describe + such a PR as benchmark-verified. Request an explicit user waiver when needed. + +The frozen benchmark is for version-to-version comparisons. Keep its cases, +data, splits and measurement semantics stable; introduce a new suite version when +these semantics change. GPU unit tests remain required and do not replace it. diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..578b704 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,162 @@ +# Delicious GPU benchmark + +The default `pyboost-delicious-paper-v1-k5` compares library revisions on the real +**Delicious** multilabel dataset from the +[SketchBoost paper, Appendix B](https://arxiv.org/html/2211.12858v1#A2). +It follows the paper's evaluation protocol and chosen hyperparameters, without +rerunning hyperparameter search or competing libraries. The full comparison can take days on one GPU, especially the unsketched baseline +with separate memory fits; the old ten-minute synthetic budget no longer applies. + +## Frozen data and protocol + +Delicious has **16105 rows, 500 features and 983 binary labels**. Download the +original 1.3 MB archive selected by the +[paper's download script](https://github.com/sb-ai-lab/SketchBoost-paper/blob/e479fde7a0dd5f624fd9efd654e2d192a12d325d/get_data.sh), +not Delicious-200K. The harness checks the archive and every input file's SHA-256 +before parsing dense float32 arrays. The NPZ cache is loaded with +`allow_pickle=False`; no external pickle is loaded. Preparation is outside timing. + +Following the [reference preprocessing](https://github.com/sb-ai-lab/SketchBoost-paper/blob/e479fde7a0dd5f624fd9efd654e2d192a12d325d/preprocess.py), +use column zero of the official split files, preserving order and converting +one-based indices. There are **12920 train / 3185 test** rows. Following the +[reference runner](https://github.com/sb-ai-lab/SketchBoost-paper/blob/e479fde7a0dd5f624fd9efd654e2d192a12d325d/runner.py), +apply `KFold(5, shuffle=True, random_state=42)` only to train: every fold has +**10336 train / 2584 validation** rows. All five models are independently evaluated +on the untouched 3185-row test set. Do not average their predictions into an +ensemble. Model seed is 42; CPU thread count is eight. + +Archive SHA-256: +`9a4e79e9bd1835225e59eef09ba0ee29c5430f785045a9245943511836542806`. +Normalized float32 data and ordered split SHA-256: +`a53ca49cdce00c38f90225994ba01e6b352e9445d121df35fcd177a1cbd5c676`. +Every worker verifies the normalized cache and records a fold checksum covering +the actual dataset plus ordered train, validation and test indices. + +The frozen manifest is `suite-delicious-paper-v1-k5.json`: + +| Parameter | Value | +|---|---:| +| Loss / validation metric | binary cross-entropy over all labels and rows | +| Maximum iterations | 20000 | +| Learning rate | 0.015 | +| Early stopping patience | 500 | +| Maximum depth | 12 | +| Maximum bins | 256 | +| Minimum data in leaf | 76 | +| Row sampling | 0.623613094287341 | +| L2 leaf regularization | 13.057198025783617 | +| Column sampling | 1.0 | +| `use_hess` | false | + +The unrounded parameters are copied into plain JSON from the Delicious/CatBoost +entry of the paper's +[`runs/baselines_and_params.pkl`](https://github.com/sb-ai-lab/SketchBoost-paper/blob/e479fde7a0dd5f624fd9efd654e2d192a12d325d/runs/baselines_and_params.pkl). +Sharing CatBoost's parameters with SketchBoost and disabling Hessians in split +scoring follows the experiment code. The tested checkout supplies the actual +sketch implementations: this compares library revisions, rather than pinning the +2022 library. No hyperparameter search runs during PR checks. + +| Method | Outputs K | Sketch width k | Coverage | +|---|---:|---:|---| +| Full GradientBoosting, single output group | 983 | all 983 | Paper baseline: no sketch | +| SketchBoost/topk | 983 | 5 | Paper: Top Outputs | +| SketchBoost/rand | 983 | 5 | Paper: Random Sampling | +| SketchBoost/proj | 983 | 5 | Paper: Random Projections | +| SketchBoost/filter | 983 | 5 | Additional library coverage, outside the paper | + +There are **25 cases**: five methods × five folds. `sketch_outputs=5` affects only +structure search; leaves and predictions retain all 983 labels. `topk` chooses +five distinct labels; `rand`/`filter` make five draws with replacement (at most +five distinct labels); `proj` creates five components. The paper also sweeps other +k values; this required profile fixes k=5 as requested. + +## Run and resume + +Use Linux, Python 3.12 and CUDA 12 with the pinned dependency closure. The harness +is independent of tested checkouts, selected by `--source`. + +```bash +uv venv --python 3.12 .venv-benchmark +uv pip install --python .venv-benchmark/bin/python -r benchmarks/requirements-py312-cu12.txt +uv pip install --python .venv-benchmark/bin/python --no-deps -e . +.venv-benchmark/bin/python benchmarks/run.py prepare-data +.venv-benchmark/bin/python benchmarks/run.py run \ + --source /path/to/base --source /path/to/candidate \ + --device 1 --output .benchmark-results/delicious-comparison +``` + +For offline preparation, use `prepare-data --archive /path/to/Delicious.zip`. +A different archive is rejected. Pass `--data-dir /path/to/cache` to preparation +and run when using a nondefault cache. Workers never download data implicitly. + +Exactly **one physical GPU** runs both versions, all methods/folds and measurement +phases sequentially. Select an index or UUID; only that UUID is exposed as logical +CUDA device zero. Multiple visible GPUs or other compute processes on the GPU +are rejected. Reports include model, physical/CUDA-visible VRAM, UUID, PCI bus, +compute capability, SM count, memory bus width, power, maximum SM clock and +NVIDIA/CUDA driver/runtime versions. Use the same idle GPU and environment for +both versions. GitHub Actions does not need a GPU worker. + +Commit library changes before running. Reports record source SHAs, imported paths +and source checksums; cases alternate version order. Each method/fold is saved in +`//results.json` alongside raw worker jobs, results and logs. +Default timeout is **72 hours per worker**, including warmup, timing and memory. +Failed cases, OOM and timeouts are failures, never skipped passes. + +Repeat the same command with `--resume` to skip completed cases and retry unfinished +or failed cases. It refuses different source SHAs, suite, harness, environment, +case selection or repeat counts. Stop the original run before resuming. Without +`--resume`, output must be empty. Per-worker logs show progress every 500 iterations. + +`--case delicious-proj-fold0` is a diagnostic subset and never fulfills the full +requirement. `--repeats` changes repetitions **within each fold**; the default is +**one**, not five. All five CV folds are required. + +## Measurement and publication + +Each version/method/fold runs in a fresh subprocess. Warmup uses 32 iterations on +that fold and is excluded from timing. It does not run 20000 warmup iterations. + +1. Fit one fresh model with up to 20000 iterations and validation early stopping. + Time fit and test prediction separately using CUDA synchronization and + `perf_counter`. Fit includes quantization, transfers and validation. +2. Score test predictions outside timers: binary cross-entropy averaged over + labels/rows, clipping probabilities to `[1e-7, 1-1e-7]` as in the paper. Use + bounded float64 accumulation. Test data never control stopping. +3. Fit and predict with another fresh model for memory measurement. Hooks and RSS + sampling are disabled during timing. Record peak live rounded CuPy allocations + (including Py-Boost's private fit pool) and CPU RSS sampled every 5 ms. GPU peaks + exclude context, free pool cache and external allocations; they are not total + process VRAM. CPU RSS includes resident input data and can miss short peaks. + +The summary reports **five-fold means ± sample standard deviations (`ddof=1`)** for fit, +prediction and test log loss, plus maximum fit GPU allocation across folds. JSON +preserves each fold's timing/memory, test quality, executed iterations and retained +trees for both phases. Folds are different datasets, not five repeats of one fit. +The per-fold table and iteration/tree counts follow the summary. Faster convergence +must be assessed together with work performed and quality. + +Two-source runs generate `comparison.md`. Saved reports can be compared with: + +```bash +.venv-benchmark/bin/python benchmarks/run.py compare \ + .benchmark-results/run/BASE_SHA/results.json \ + .benchmark-results/run/CANDIDATE_SHA/results.json \ + --output .benchmark-results/run/comparison.md +``` + +Comparison rejects mismatched workloads, datasets, environments and failed runs. +Partial runs are marked `full_suite: false`. Measurements are not automatic +statistical verdicts or a claim of exactly matching the paper's scores/timings on +different hardware/software. Repeated runs can vary with GPU clocks and temperature. + +Publish the table and run configuration in a PR comment. Keep raw JSON, logs, +archives and cached data locally in ignored directories. Do not create result +branches or upload artifacts unless explicitly requested. The runner performs no +network writes; only `prepare-data` downloads the pinned archive. + +Legacy synthetic workloads remain selectable with `--suite +benchmarks/suite-v1-k5.json` or `--suite benchmarks/suite-v1.json` (k=8); see +[SYNTHETIC.md](SYNTHETIC.md). They do not fulfill the Delicious requirement. Frozen +workloads, data, splits or measurement changes require a new suite ID and rerunning +both library versions with the same final harness. diff --git a/benchmarks/SYNTHETIC.md b/benchmarks/SYNTHETIC.md new file mode 100644 index 0000000..fc93ce8 --- /dev/null +++ b/benchmarks/SYNTHETIC.md @@ -0,0 +1,173 @@ +# Py-Boost GPU benchmarks + +`pyboost-gpu-v1-k5` is the legacy synthetic cross-version benchmark, not a collection of ad hoc +experiments. Run this harness against clean Git checkouts using the same Python +environment and one otherwise idle NVIDIA GPU. A GitHub Actions GPU runner is +not required. The tested library version is selected by `--source`, independently +of the directory containing the harness. + +## Frozen workload + +Definitions are in `suite-v1-k5.json`. All cases build **32 iterations**, use learning +rate 0.1, seed 42, full row/column sampling, and `use_hess=True`. Defaults are depth +4 and 64 bins. Every sketch case explicitly sets **`sketch_outputs=5`**. +Here **K** is the original number of outputs and **k=5** is the sketch width used +only for structure search; final predictions still contain all K outputs: + +- `proj`: five projected components combining the original outputs. +- `rand`: five output draws with replacement, so at most five distinct outputs. +- `filter`: five importance-weighted output draws with replacement, so at most + five distinct outputs. +- `topk`: five distinct outputs with the largest gradient norms. +- Full `GradientBoosting`: all K outputs, without a sketch. + +Early stopping is disabled. The benchmark counts executed iterations and +separately records retained +trees: existing Py-Boost versions can prune to the best validation iteration even +with `es=0`. Different retained tree counts are disclosed in comparisons. + +| Case | Train / validation / test rows | Features | Outputs K | Sketch width k | Model | +|---|---:|---:|---:|---:|---| +| mse-single | 16384 / 4096 / 4096 | 32 | 1 | all | GradientBoosting | +| mse-full-128 | 8192 / 2048 / 2048 | 32 | 128 | all | GradientBoosting | +| mse-proj-32 | 16384 / 4096 / 4096 | 32 | 32 | 5 | SketchBoost/proj | +| mse-proj-128 | 16384 / 4096 / 4096 | 32 | 128 | 5 | SketchBoost/proj | +| mse-proj-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 5 | SketchBoost/proj | +| mse-proj-10000 | 16384 / 4096 / 4096 | 32 | 10000 | 5 | SketchBoost/proj | +| bce-rand-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 5 | SketchBoost/rand | +| bce-filter-1024 | 16384 / 4096 / 4096 | 32 | 1024 | 5 | SketchBoost/filter | +| mse-topk-10000 | 4096 / 1024 / 1024 | 32 | 10000 | 5 | SketchBoost/topk | +| ce-proj-512 | 16384 / 4096 / 4096 | 32 | 512 classes | 5 | SketchBoost/proj | +| validation-proj-1024 | 8192 / 32768 / 4096 | 32 | 1024 | 5 | SketchBoost/proj; two validation sets | +| histogram-topk-128 | 8192 / 2048 / 2048 | 128 | 128 | 5 | SketchBoost/topk; depth 8, 256 bins | + +The four `mse-proj-*` cases change only the number of outputs. Other cases exercise +full-output scoring, separate/shared Hessians, wide targets, validation memory, +and deep histograms. No sketch method is treated as the primary benchmark. +SVD requires an additional RAPIDS stack and is outside v1. Missing-label losses +are outside v1 to keep this suite runnable against released 0.5.2 as well as newer +versions; add such workloads under a new suite ID. + +Data are generated by `protocol.make_data` using PCG64 seed 20261030, float32 +features/targets and eight correlated output groups. Regression has additive +noise; multilabel targets are Bernoulli draws from signal-dependent probabilities. +Multiclass labels encode feature signs, with all classes represented in training. +Generation uses no matrix multiplication or external downloads. SHA-256 covers +the actual feature and target bytes. Validation/test rows are held out. The rand +and filter cases receive identical data. + +## Environment and commands + +The reference stack is Linux, Python 3.12 and CUDA 12. Install the complete pinned +dependency closure in a dedicated environment. This file deliberately does not +install a particular Py-Boost revision; install metadata once, then select each +tested checkout through `--source`. + +```bash +uv venv --python 3.12 .venv-benchmark +uv pip install --python .venv-benchmark/bin/python -r benchmarks/requirements-py312-cu12.txt +uv pip install --python .venv-benchmark/bin/python --no-deps -e . +git worktree add --detach ../Py-Boost-v0.5.2 v0.5.2 +.venv-benchmark/bin/python benchmarks/run.py run \ + --suite benchmarks/suite-v1-k5.json \ + --source ../Py-Boost-v0.5.2 --source . \ + --device 1 --output .benchmark-results/v052-vs-current +``` + +The command above explicitly selects synthetic data with `sketch_outputs=5`. To reproduce the original +`pyboost-gpu-v1` workload with `sketch_outputs=8`, explicitly select its unchanged +manifest. Reports with different sketch widths are not comparable: + +```bash +.venv-benchmark/bin/python benchmarks/run.py run \ + --suite benchmarks/suite-v1.json \ + --source /path/to/baseline --source /path/to/candidate \ + --device 1 --output .benchmark-results/k8-comparison +``` + +Commit changes to `py_boost/` or `pyproject.toml` first: dirty library sources are +rejected. **Exactly one physical GPU is used for the entire run.** Use a physical +GPU index or UUID (`nvidia-smi -L`). `--device 1` selects only physical GPU #1; it +is resolved to its physical UUID and exposed inside each worker as logical +`cuda:0` through `CUDA_VISIBLE_DEVICES`. UUID selection avoids differences between +CUDA and `nvidia-smi` ordinal ordering. +A comma-separated GPU list is rejected, and workers verify that exactly one CUDA +device is visible. Cases, versions and measurement phases run sequentially. +Do not share the selected GPU with another workload. Preflight performs a real allocation/kernel, rejects other +compute processes on the GPU, and records its UUID. CPU thread counts are fixed at +4. No GPU is a failure, not a skipped or CPU-emulated benchmark. + +Each source produces `//results.json` and raw worker +requests/results/logs. GPU count, model, physical and CUDA-visible VRAM, UUID, +PCI bus, compute capability, SM count, memory bus width, power limit, maximum SM +clock and NVIDIA/CUDA driver/runtime versions are recorded +in JSON and displayed in the comparison report. Each comparison row shows the +full output count K and sketch width k/method explicitly. +Two-source runs also produce `comparison.md`. More than two +sources may be measured in one run; compare any two results afterward: + +```bash +.venv-benchmark/bin/python benchmarks/run.py compare \ + .benchmark-results/older/COMMIT/results.json \ + .benchmark-results/newer/COMMIT/results.json \ + --output .benchmark-results/comparison.md +``` + +`--python /path/to/environment/bin/python` selects the worker interpreter. +`--case mse-single --repeats 1` is a diagnostic subset of this legacy +profile. Output directories must be empty so previous evidence is not overwritten. +Workers have a 900-second timeout including setup/warmup and both phases; failures and OOM retain +logs/results, and the command exits nonzero. A version lacking a required API is +reported as failed, not silently replaced by another implementation. + +## Measurement contract + +- Each case/version uses a fresh subprocess. When multiple + versions are measured, their order alternates by case. +- Each subprocess generates and hashes data, then warms one complete fit/predict. + Data generation, imports, JIT warmup and estimator construction are excluded + from timing. Quantization, CPU/GPU transfers and evaluation inside `fit()` are + included. Prediction receives CPU input and includes transfers back to CPU. +- Timing uses five fresh-model repetitions, `perf_counter` and device-wide CUDA + synchronization. Raw samples, median and p10/p90 are retained. These are warmed + timings, not cold-start latency. Holdout scoring is outside the timers. +- After the timing repetitions, memory uses a separate fresh-model fit/predict in + the same subprocess. Hooks/RSS sampling are disabled during timing. Keeping + both phases in one process avoids repeating Numba import-time compilation. + CuPy malloc/free hooks track **peak live rounded allocation bytes**, including + Py-Boost's private fit pool. This excludes free pool cache, CUDA context and + allocations outside CuPy's hooked pools; it is not total process VRAM. + Prediction tracking includes still-live allocations observed during fit. +- CPU RSS is sampled every 5 ms during fit/predict and includes the resident input + dataset. It is an approximation and may miss very short peaks. Absolute peaks + and the fit baseline are stored. +- Holdout RMSE or log loss is computed in bounded float64 blocks. Every prediction + must be finite and have the expected shape. Metric samples, actual iteration + counts and retained tree counts are stored, so faster runs can be assessed + together with model quality and work performed. + +## Cross-version comparability and publication + +Reports identify the exact library commit and library-source checksum, benchmark +implementation checksum, manifest checksum, actual dataset checksum, GPU UUID, +CPU/OS, Python, relevant dependency versions, CUDA driver/runtime, repetitions and +warmups. `compare` refuses mismatched workloads, datasets, environments or failed +measurements (exit code 2). Partial runs are marked `full_suite: false`; only a full +five-repeat run completes this legacy profile. The current PR requirement uses +Delicious; see README.md. Comparison tables show measured +changes, not an automatic statistical pass/fail verdict. + +To compare historical versions, run them through **this same harness** and pinned +environment. Different GPUs/environments need separate baselines. Repeated runs +can still differ due to temperature, clocks and scheduling: retain raw samples +and investigate noise before calling a small difference a regression. + +Treat both profiles' data, cases and measurement semantics as frozen. New semantics/workloads +need a new suite version. Any harness change alters its checksum and requires +remeasuring both versions; never relabel old results to bypass that check. + +Publish the comparison table and run configuration in a PR comment. Retain both +`results.json` files, raw logs and archives locally in the ignored output +directory. Do not create result branches or upload artifacts unless explicitly +requested. The runner itself performs no network writes. Do not commit local +results or environment directories into the code branch. diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..2252207 --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1 @@ +"""Versioned, standalone benchmarks; not part of the py_boost package.""" diff --git a/benchmarks/delicious.py b/benchmarks/delicious.py new file mode 100644 index 0000000..4b32957 --- /dev/null +++ b/benchmarks/delicious.py @@ -0,0 +1,105 @@ +"""Pinned Delicious data and the five-fold evaluation protocol from SketchBoost. + +No pickled data or models are downloaded or deserialized by this harness. +""" + +import hashlib +from pathlib import Path +import tempfile +from urllib.request import urlopen +from zipfile import ZipFile + +import numpy as np + +DATA_URL = 'https://drive.google.com/uc?export=download&id=0B3lPMIHmG6vGdG1jZ19VS2NWRVU' +ARCHIVE_SHA256 = '9a4e79e9bd1835225e59eef09ba0ee29c5430f785045a9245943511836542806' +FILES = { + 'Delicious/Delicious_data.txt': '7c6732b0ba3e2acb8b6f8cd147b83107f798e50fedf47c3429f549d53d971b9b', + 'Delicious/delicious_trSplit.txt': '8dbe2f693e62ae6e45ee617db8372fd5f547f4bb2f59e17c47f55c97b7977d1e', + 'Delicious/delicious_tstSplit.txt': 'aac14f9f84e6fe3f77d81db8ceb04be935e72adf22d9284d78d58d7763f242fa', +} +DEFAULT_DATA_DIR = Path(__file__).resolve().parents[1] / '.benchmark-data' + + +def array_hash(*arrays): + digest = hashlib.sha256() + for arr in arrays: + arr = np.ascontiguousarray(arr) + digest.update(str((arr.shape, arr.dtype.str)).encode()) + digest.update(memoryview(arr).cast('B')) + return digest.hexdigest() + + +def read_archive(archive): + if hashlib.sha256(Path(archive).read_bytes()).hexdigest() != ARCHIVE_SHA256: + raise ValueError('Delicious archive SHA-256 mismatch') + with ZipFile(archive) as zipped: + contents = {} + for name, expected in FILES.items(): + value = zipped.read(name) + if hashlib.sha256(value).hexdigest() != expected: + raise ValueError(f'Delicious file SHA-256 mismatch: {name}') + contents[name] = value.decode('ascii').splitlines() + lines = contents['Delicious/Delicious_data.txt'] + if lines[0].split() != ['16105', '500', '983'] or len(lines) != 16106: + raise ValueError('Unexpected Delicious dimensions') + X = np.zeros((16105, 500), dtype=' 16 * 1024 * 1024: + raise ValueError('Unexpected Delicious archive size') + output.write(block) + arrays = read_archive(archive) + prepared = Path(temporary) / 'delicious.npz' + np.savez_compressed(prepared, **dict(zip(['X', 'y', 'train', 'test'], arrays))) + prepared.replace(destination) + return destination, array_hash(*arrays) + + +def load_fold(case, seed, data_dir): + from sklearn.model_selection import KFold + + path = Path(data_dir or DEFAULT_DATA_DIR) / 'delicious.npz' + if not path.is_file(): + raise ValueError(f'Missing {path}; run benchmarks/run.py prepare-data first') + with np.load(path, allow_pickle=False) as cache: + X, y, train, test = [cache[name] for name in ['X', 'y', 'train', 'test']] + # Validate the actual normalized arrays every time, including split ordering. + if array_hash(X, y, train, test) != NORMALIZED_SHA256: + raise ValueError('Prepared Delicious dataset checksum mismatch; prepare it again') + folds = list(KFold(5, shuffle=True, random_state=seed).split(train)) + fit, valid = [train[indices] for indices in folds[case['fold']]] + if [len(fit), len(valid), len(test)] != [case['train'], case['valid'], case['test']]: + raise ValueError('Delicious fold sizes differ from manifest') + digest = array_hash(X, y, fit, valid, test) + return X[fit], y[fit], [{'X': X[valid], 'y': y[valid]}], X[test], y[test], digest + + +# Computed from the checked archive, float32 arrays and ordered official splits. +NORMALIZED_SHA256 = 'a53ca49cdce00c38f90225994ba01e6b352e9445d121df35fcd177a1cbd5c676' diff --git a/benchmarks/gpu.py b/benchmarks/gpu.py new file mode 100644 index 0000000..5ba8d59 --- /dev/null +++ b/benchmarks/gpu.py @@ -0,0 +1,50 @@ +"""Measurement helpers. GPU timing and allocation tracking run separately.""" + +import threading + + +def memory_tracker(cp): + class Tracker(cp.cuda.MemoryHook): + def __init__(self): + self.live = {} + self.current = 0 + self.peak = 0 + + def malloc_postprocess(self, device_id, size, mem_size, mem_ptr, pmem_id): + if mem_ptr and pmem_id: + key = (device_id, pmem_id) + self.current += mem_size - self.live.get(key, 0) + self.live[key] = mem_size + self.peak = max(self.peak, self.current) + + def free_postprocess(self, device_id, mem_size, mem_ptr, pmem_id): + self.current -= self.live.pop((device_id, pmem_id), 0) + + def reset_peak(self): + self.peak = self.current + + return Tracker() + + +class RssPeak: + """Sample absolute RSS during one phase; includes resident CPU input data.""" + + def __init__(self, process): + self.process = process + self.stop = threading.Event() + + def __enter__(self): + self.baseline = self.process.memory_info().rss + self.peak = self.baseline + self.thread = threading.Thread(target=self._sample, daemon=True) + self.thread.start() + return self + + def _sample(self): + while not self.stop.wait(.005): + self.peak = max(self.peak, self.process.memory_info().rss) + + def __exit__(self, *exc): + self.peak = max(self.peak, self.process.memory_info().rss) + self.stop.set() + self.thread.join() diff --git a/benchmarks/protocol.py b/benchmarks/protocol.py new file mode 100644 index 0000000..bede891 --- /dev/null +++ b/benchmarks/protocol.py @@ -0,0 +1,142 @@ +"""Data and result contracts shared by every tested library version.""" + +import hashlib +import json +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parent +DEFAULT_SUITE = ROOT / 'suite-delicious-paper-v1-k5.json' +SCHEMA_VERSION = 1 + + +def digest_json(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest() + + +def implementation_hash(suite=None): + digest = hashlib.sha256() + digest.update(digest_json(load_suite() if suite is None else suite).encode()) + for name in ['protocol.py', 'gpu.py', 'run.py', 'delicious.py']: + digest.update(name.encode()) + digest.update((ROOT / name).read_bytes()) + return digest.hexdigest() + + +def load_suite(path=None): + return json.loads((Path(path) if path is not None else DEFAULT_SUITE).read_text()) + + +def make_data(case, seed, data_dir=None): + """PCG64, float32 inputs, eight correlated output groups, no external data.""" + if case.get('dataset') == 'delicious': + if __package__: + from .delicious import load_fold + else: + from delicious import load_fold + return load_fold(case, seed, data_dir) + rng = np.random.Generator(np.random.PCG64(seed)) + n = case['train'] + case['valid'] + case['test'] + X = rng.standard_normal((n, case['features']), dtype=np.float32) + outputs = case['outputs'] + if case['task'] == 'multiclass': + bits = (outputs - 1).bit_length() + y = np.zeros(n, dtype=np.int32) + for bit in range(bits): + y |= (X[:, bit] > 0).astype(np.int32) << bit + y %= outputs + # Ensure every class is represented in training, with consistent features. + for bit in range(bits): + X[:outputs, bit] = np.where((np.arange(outputs) >> bit) & 1, .5, -.5) + y[:outputs] = np.arange(outputs) + else: + latent = np.stack([X[:, 0], X[:, 1], X[:, 2] * X[:, 3], + np.maximum(X[:, 4], 0), np.sin(X[:, 5]), + X[:, 6] + X[:, 7], X[:, 8] - X[:, 9], + (X[:, 10] > 0).astype(np.float32) - .5], axis=1) + y = np.empty((n, outputs), dtype=np.float32) + for start in range(0, outputs, 128): + stop = min(start + 128, outputs) + columns = np.arange(start, stop) + signal = latent[:, columns % 8] + .3 * latent[:, (columns + 3) % 8] + noise = rng.standard_normal(signal.shape, dtype=np.float32) + if case['task'] == 'regression': + y[:, start:stop] = signal + .1 * noise + else: + probability = 1 / (1 + np.exp(-signal)) + y[:, start:stop] = rng.random(signal.shape, dtype=np.float32) < probability + digest = hashlib.sha256() + for arr in [X, y]: + digest.update(str((arr.shape, arr.dtype.str)).encode()) + digest.update(memoryview(arr).cast('B')) + t, v = case['train'], case['train'] + case['valid'] + validation = [] + for indices in np.array_split(np.arange(t, v), case['validation_sets']): + validation.append({'X': X[indices[0]:indices[-1] + 1], + 'y': y[indices[0]:indices[-1] + 1]}) + return X[:t], y[:t], validation, X[v:], y[v:], digest.hexdigest() + + +def quality(task, target, prediction, clip=1e-15): + """Holdout metric outside timed regions, with bounded float64 temporaries.""" + if not np.isfinite(prediction).all(): + raise ValueError('Non-finite predictions') + if prediction.shape[0] != target.shape[0]: + raise ValueError('Prediction row count differs from target') + total, count = 0., 0 + for start in range(0, len(target), 256): + y = target[start:start + 256] + p = prediction[start:start + 256].astype(np.float64) + if task == 'multiclass': + if p.ndim != 2 or (y.max() >= p.shape[1]): + raise ValueError('Missing class probabilities') + values = -np.log(np.clip(p[np.arange(len(y)), y], 1e-15, 1)) + elif task == 'regression': + if p.shape != y.shape: + raise ValueError('Prediction output count differs from target') + values = (p - y.astype(np.float64)) ** 2 + else: + if p.shape != y.shape: + raise ValueError('Prediction output count differs from target') + p = np.clip(p, clip, 1 - clip) + values = -(y * np.log(p) + (1 - y) * np.log1p(-p)) + total += float(values.sum()) + count += values.size + value = total / count + return {'name': 'rmse' if task == 'regression' else 'logloss', + 'value': float(np.sqrt(value) if task == 'regression' else value), + 'lower_is_better': True} + + +def summarize(values): + return {'median': float(np.median(values)), 'p10': float(np.quantile(values, .1)), + 'p90': float(np.quantile(values, .9)), 'samples': values} + + +def comparison_errors(base, head): + errors = [] + for key in ['schema_version', 'benchmark_id', 'implementation_hash', 'suite_hash', + 'case_ids', 'repeats', 'warmups']: + if base.get(key) != head.get(key): + errors.append(f'Different {key}') + if base.get('environment') != head.get('environment'): + errors.append('Different hardware or software environment') + for label, report in [('base', base), ('candidate', head)]: + if not report.get('case_ids') or set(report.get('case_ids', [])) != set(report.get('cases', {})): + errors.append(f'{label}: missing case results') + for case_id in sorted(set(base.get('cases', {})) & set(head.get('cases', {}))): + a, b = base['cases'][case_id], head['cases'][case_id] + if a.get('status') != 'ok' or b.get('status') != 'ok': + errors.append(f'{case_id}: failed or incomplete measurement') + elif a.get('data_hash') != b.get('data_hash'): + errors.append(f'{case_id}: different dataset checksum') + elif not a.get('data_hash'): + errors.append(f'{case_id}: missing dataset checksum') + if a.get('spec') != b.get('spec'): + errors.append(f'{case_id}: different case parameters') + if any(c.get('status') != 'ok' for c in list(base.get('cases', {}).values()) + list(head.get('cases', {}).values())): + errors.append('One or more cases did not complete') + if not base.get('complete') or not head.get('complete'): + errors.append('A report is incomplete') + return errors diff --git a/benchmarks/requirements-py312-cu12.txt b/benchmarks/requirements-py312-cu12.txt new file mode 100644 index 0000000..6e4cca6 --- /dev/null +++ b/benchmarks/requirements-py312-cu12.txt @@ -0,0 +1,32 @@ +# Tested Linux/Python 3.12/CUDA 12 dependency closure; library checkout is selected separately. +cloudpickle==3.1.2 +cuda-pathfinder==1.8.5 +cupy-cuda12x==14.2.0 +flatbuffers==25.12.19 +iniconfig==2.3.1 +joblib==1.6.0 +llvmlite==0.50.0 +ml-dtypes==0.6.0 +narwhals==2.28.0 +numba==0.68.1 +numpy==2.5.4 +onnx==1.23.2 +onnxruntime==1.31.1 +packaging==26.3 +pandas==3.1.0 +pluggy==1.7.0 +protobuf==7.36.2 +psutil==7.2.2 +pygments==2.21.0 +pytest==9.2.0 +python-dateutil==3.0.0 +scikit-learn==1.9.1 +scipy==1.18.1 +six==1.17.0 +threadpoolctl==3.7.0 +tqdm==4.70.1 +treelite==3.9.1 +treelite-runtime==3.9.1 +typing-extensions==4.16.0 +tzdata==2026.6 +ujson==6.0.0 diff --git a/benchmarks/run.py b/benchmarks/run.py new file mode 100644 index 0000000..50b67df --- /dev/null +++ b/benchmarks/run.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +"""Run a frozen GPU suite against one or more Py-Boost source checkouts.""" + +import argparse +import datetime +import gc +import hashlib +import importlib.metadata +import json +import logging +import os +from pathlib import Path +import platform +import subprocess +import sys +import time +import traceback + +if __package__: + from .protocol import (DEFAULT_SUITE, ROOT, SCHEMA_VERSION, comparison_errors, digest_json, + implementation_hash, load_suite, make_data, quality, summarize) + from .gpu import RssPeak, memory_tracker +else: + from protocol import (DEFAULT_SUITE, ROOT, SCHEMA_VERSION, comparison_errors, digest_json, + implementation_hash, load_suite, make_data, quality, summarize) + from gpu import RssPeak, memory_tracker + + +def write_json(path, data): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + '.tmp') + temporary.write_text(json.dumps(data, indent=2, sort_keys=True, allow_nan=False) + '\n') + temporary.replace(path) + + +def git(source, *args): + return subprocess.check_output(['git', '-C', str(source), *args], text=True).strip() + + +def source_info(source): + source = Path(source).resolve() + if not (source / 'py_boost' / '__init__.py').is_file(): + raise ValueError(f'Not a Py-Boost checkout: {source}') + paths = ['py_boost', 'pyproject.toml'] + if git(source, 'status', '--porcelain', '--untracked-files=all', '--', *paths): + raise ValueError(f'Commit library changes before benchmarking: {source}') + digest = hashlib.sha256() + for name in git(source, 'ls-files', '--', *paths).splitlines(): + digest.update(name.encode()) + digest.update((source / name).read_bytes()) + return {'path': str(source), 'commit': git(source, 'rev-parse', 'HEAD'), + 'library_hash': digest.hexdigest()} + + +def resolve_gpu(selector): + """Use a physical UUID: CUDA and nvidia-smi ordinal ordering can differ.""" + devices = subprocess.check_output( + ['nvidia-smi', '-i', selector, '--query-gpu=uuid', '--format=csv,noheader'], + text=True).strip().splitlines() + if len(devices) != 1 or not devices[0].startswith('GPU-'): + raise ValueError('Select exactly one physical GPU index or UUID') + return devices[0] + + +def environment(cp, np, psutil, device): + if cp.cuda.runtime.getDeviceCount() != 1: + raise RuntimeError('Exactly one visible CUDA device is required') + cp.zeros(1).sum().item() # A real allocation and kernel; no CPU fallback. + props = cp.cuda.runtime.getDeviceProperties(0) + name = props['name'] + gpu_details = subprocess.check_output( + ['nvidia-smi', '-i', device, + '--query-gpu=uuid,driver_version,pci.bus_id,memory.total,power.limit,clocks.max.sm', + '--format=csv,noheader,nounits'], text=True).strip() + if '\n' in gpu_details: + raise RuntimeError('GPU selector resolved to more than one physical device') + uuid, driver_version, pci_bus_id, memory_mib, power_limit, max_sm_clock = [ + part.strip() for part in gpu_details.split(',')] + processes = subprocess.check_output( + ['nvidia-smi', '--query-compute-apps=gpu_uuid,pid', '--format=csv,noheader,nounits'], + text=True) + busy = [line for line in processes.splitlines() + if line.split(',')[0].strip() == uuid and line.split(',')[1].strip() != str(os.getpid())] + if busy: + raise RuntimeError(f'GPU is in use by other compute processes: {busy}') + packages = {} + for distribution in importlib.metadata.distributions(): + package = distribution.metadata['Name'] + if package and package.lower().replace('_', '-') in { + 'numpy', 'scipy', 'numba', 'llvmlite', 'scikit-learn', 'joblib', 'pandas', + 'cupy', 'cupy-cuda11x', 'cupy-cuda12x', 'cupy-cuda13x', 'psutil', + 'treelite', 'treelite-runtime', 'onnx', 'onnxruntime', 'ujson', + }: + packages[package.lower().replace('_', '-')] = distribution.version + cpu = platform.processor() + if Path('/proc/cpuinfo').exists(): + cpu = next((line.split(':', 1)[1].strip() for line in + Path('/proc/cpuinfo').read_text().splitlines() if line.startswith('model name')), cpu) + return {'python': platform.python_version(), 'platform': platform.platform(), + 'cpu': cpu, 'cpu_threads': int(os.environ['NUMBA_NUM_THREADS']), + 'host_ram_bytes': psutil.virtual_memory().total, 'gpu_count': 1, + 'gpu': {'name': name.decode() if isinstance(name, bytes) else name, + 'uuid': uuid, 'vram_bytes': props['totalGlobalMem'], + 'driver_version': driver_version, 'pci_bus_id': pci_bus_id, + 'physical_vram_bytes': int(float(memory_mib) * 2**20), + 'power_limit_watts': power_limit, 'max_sm_clock_mhz': max_sm_clock, + 'multiprocessors': props['multiProcessorCount'], + 'memory_bus_width_bits': props.get('memoryBusWidth'), + 'compute_capability': [props['major'], props['minor']]}, + 'cuda_driver': cp.cuda.runtime.driverGetVersion(), + 'cuda_runtime': cp.cuda.runtime.runtimeGetVersion(), 'packages': packages} + + +def worker(job): + import numpy as np + import cupy as cp + import psutil + + logging.disable(logging.CRITICAL) + if job.get('implementation_hash', implementation_hash(job['suite'])) != implementation_hash(job['suite']): + raise RuntimeError('Benchmark harness changed after the run started') + cp.cuda.Device(0).use() + metadata = environment(cp, np, psutil, job['device']) + if job['phase'] == 'metadata': + return {'status': 'ok', 'environment': metadata} + sys.path.insert(0, job['source']['path']) + import py_boost + expected = Path(job['source']['path']) + if Path(py_boost.__file__).resolve().parents[1] != expected: + raise RuntimeError(f'Imported the wrong library: {py_boost.__file__}') + if source_info(expected) != job['source']: + raise RuntimeError('Library source changed after the run started') + case, suite = job['case'], job['suite'] + X, y, valid, test_X, test_y, data_hash = make_data(case, suite['data_seed'], job.get('data_dir')) + params = dict(suite['common_params'], **case['params']) + params.update(loss=case['loss'], seed=suite['model_seed']) + if case['model'] == 'SketchBoost': + params['sketch_params'] = suite['sketch_params'][params['sketch_method']] + else: + params['target_splitter'] = 'Single' + + class Iterations(py_boost.Callback): + count = 0 + + def after_iteration(self, info): + self.count += 1 + if self.count % 500 == 0: + print(f'{case["id"]}: {self.count} iterations', flush=True) + return False + + def new_model(overrides=None): + counter = Iterations() + model = getattr(py_boost, case['model'])(**dict(params, **(overrides or {})), callbacks=[counter]) + return model, counter + + def cleanup(model): + model.to_cpu() + gc.collect() + cp.get_default_memory_pool().free_all_blocks() + cp.get_default_pinned_memory_pool().free_all_blocks() + + def sync(): + cp.cuda.Device(0).synchronize() + + for _ in range(suite['warmups']): + warm, _ = new_model({'ntrees': suite['warmup_iterations']} if 'warmup_iterations' in suite else None) + warm.fit(X, y, eval_sets=valid) + warm.predict(test_X) + sync() + cleanup(warm) + del warm + gc.collect() + cp.get_default_memory_pool().free_all_blocks() + def measure(phase): + fit_times, predict_times, measurements = [], [], [] + repeats = job['repeats'] if phase == 'timing' else 1 + for _ in range(repeats): + model, counter = new_model() + if phase == 'timing': + sync() + start = time.perf_counter() + model.fit(X, y, eval_sets=valid) + sync() + fit_times.append(time.perf_counter() - start) + start = time.perf_counter() + prediction = model.predict(test_X) + sync() + predict_times.append(time.perf_counter() - start) + else: + with memory_tracker(cp) as tracker: + with RssPeak(psutil.Process()) as rss_fit: + model.fit(X, y, eval_sets=valid) + sync() + fit_peak = tracker.peak + tracker.reset_peak() + with RssPeak(psutil.Process()) as rss_predict: + prediction = model.predict(test_X) + sync() + memory = {'fit_cupy_peak_bytes': fit_peak, + 'predict_cupy_peak_bytes': tracker.peak, + 'fit_rss_peak_bytes': rss_fit.peak, + 'predict_rss_peak_bytes': rss_predict.peak, + 'fit_rss_baseline_bytes': rss_fit.baseline} + if not 1 <= counter.count <= params['ntrees']: + raise RuntimeError(f'Invalid executed iteration count: {counter.count}') + if not params['es'] and counter.count != params['ntrees']: + raise RuntimeError(f'Expected {params["ntrees"]} iterations, got {counter.count}') + if prediction.shape != (case['test'], case['outputs']): + raise RuntimeError(f'Unexpected prediction shape: {prediction.shape}') + score = quality(case['task'], test_y, prediction, + clip=suite.get('dataset', {}).get('metric_clip', 1e-15)) + if not np.isfinite(score['value']): + raise RuntimeError('Non-finite holdout metric') + measurements.append({'trained_iterations': counter.count, + 'retained_trees': len(model.models), 'quality': score, + 'stopped_early': counter.count < params['ntrees']}) + print(f'{case["id"]} {phase}: iterations={counter.count}, retained={len(model.models)}, ' + f'test {score["name"]}={score["value"]:.8f}', flush=True) + cleanup(model) + del model, prediction + result = {'status': 'ok', 'environment': metadata, 'data_hash': data_hash, + 'params': params, 'measurements': measurements, + 'quality': dict(name=score['name'], **summarize([m['quality']['value'] for m in measurements])), + 'imported_library': py_boost.__file__} + if phase == 'timing': + result.update(fit_seconds=summarize(fit_times), predict_seconds=summarize(predict_times)) + else: + result.update(memory) + return result + + + result = {'status': 'running', 'environment': metadata, 'data_hash': data_hash} + for phase in ['timing', 'memory']: + try: + result[phase] = measure(phase) + except Exception: + result[phase] = {'status': 'error', 'error': traceback.format_exc()} + result['status'] = 'error' + return result + write_json(job['result_path'], result) + result['status'] = 'ok' + return result + +def invoke(python, output, job, timeout): + output = Path(output) + request, result, log = output.with_suffix('.job.json'), output.with_suffix('.json'), output.with_suffix('.log') + job['result_path'] = str(result) + write_json(request, job) + env = dict(os.environ) + env['CUDA_VISIBLE_DEVICES'] = job['device'] + for key in ['OMP_NUM_THREADS', 'OPENBLAS_NUM_THREADS', 'MKL_NUM_THREADS', 'NUMBA_NUM_THREADS']: + env[key] = str(job['suite']['cpu_threads']) + env['PYTHONHASHSEED'] = '0' + env['PYTHONPATH'] = job['source']['path'] + try: + with log.open('w') as stream: + process = subprocess.run( + [python, str(Path(__file__).resolve()), '_worker', str(request), str(result)], + stdout=stream, stderr=subprocess.STDOUT, env=env, timeout=timeout, + cwd=job['source']['path']) + if result.exists(): + record = json.loads(result.read_text()) + else: + record = {'status': 'error', 'error': f'Worker exited {process.returncode} without a result'} + if process.returncode and record.get('status') == 'ok': + record = {'status': 'error', 'error': f'Worker exited {process.returncode}'} + except subprocess.TimeoutExpired: + record = json.loads(result.read_text()) if result.exists() else {} + record.update(status='error', error=f'Timeout after {timeout} seconds') + record['log'] = str(log) + return record + + +def run(args): + suite = load_suite(args.suite) + harness_hash = implementation_hash(suite) + args.repeats = suite['repeats'] if args.repeats is None else args.repeats + args.timeout = suite.get('timeout', 900) if args.timeout is None else args.timeout + sources = [source_info(p) for p in args.source] + if len({s['commit'] for s in sources}) != len(sources): + raise ValueError('Each source must have a different commit') + selected = args.case or [c['id'] for c in suite['cases']] + if set(selected) - {c['id'] for c in suite['cases']}: + raise ValueError('Unknown case ID') + if args.repeats < 1 or args.timeout < 1: + raise ValueError('Repeats and timeout must be positive') + if ',' in args.device or not args.device: + raise ValueError('Select exactly one GPU index or UUID') + if suite.get('dataset', {}).get('name') == 'Delicious': + if not (Path(args.data_dir) / 'delicious.npz').is_file(): + raise ValueError('Run benchmarks/run.py prepare-data before benchmarking Delicious') + device = resolve_gpu(args.device) + cases = [c for c in suite['cases'] if c['id'] in selected] + output = Path(args.output).resolve() + output.mkdir(parents=True, exist_ok=True) + if any(output.iterdir()) and not args.resume: + raise ValueError('Output directory must be empty; preserve previous measurements') + reports, directories = [], [] + for source in sources: + directory = output / source['commit'][:12] + (directory / 'raw').mkdir(parents=True, exist_ok=True) + report = {'schema_version': SCHEMA_VERSION, 'benchmark_id': suite['benchmark_id'], + 'implementation_hash': harness_hash, 'suite_hash': digest_json(suite), + 'started_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'source': source, 'case_ids': [c['id'] for c in cases], + 'full_suite': len(cases) == len(suite['cases']), + 'repeats': args.repeats, 'warmups': suite['warmups'], + 'suite': suite, 'cases': {}, 'complete': False} + job = dict(source=source, phase='metadata', device=device, suite=suite, + implementation_hash=harness_hash) + probe = invoke(args.python, directory / 'raw' / 'metadata', job, args.timeout) + if probe['status'] != 'ok': + report['error'] = probe + write_json(directory / 'results.json', report) + raise RuntimeError(f'GPU/environment preflight failed; see {probe["log"]}') + report['environment'] = probe['environment'] + if args.resume and (directory / 'results.json').is_file(): + previous = json.loads((directory / 'results.json').read_text()) + for key in ['source', 'implementation_hash', 'suite_hash', 'case_ids', + 'repeats', 'warmups', 'environment']: + if previous.get(key) != report.get(key): + raise ValueError(f'Cannot resume: different {key}') + report = previous + report['complete'] = False + reports.append(report) + directories.append(directory) + write_json(directory / 'results.json', report) + for index, case in enumerate(cases): + # One fresh process per version/case; timing and memory are separate phases. + order = list(range(len(sources))) + if index % 2: + order.reverse() + for i in order: + if implementation_hash(suite) != harness_hash: + raise RuntimeError('Benchmark harness changed; preserve this run and start a new one') + if reports[i]['cases'].get(case['id'], {}).get('status') == 'ok': + continue + print(f'{sources[i]["commit"][:12]} {case["id"]}', flush=True) + job = dict(source=sources[i], phase='case', device=device, suite=suite, + case=case, repeats=args.repeats, data_dir=str(Path(args.data_dir).resolve())) + job['implementation_hash'] = harness_hash + record = invoke(args.python, directories[i] / 'raw' / case['id'], job, args.timeout) + if record['status'] == 'ok' and record['environment'] != reports[i]['environment']: + record.update(status='error', error='Environment changed during the run') + reports[i]['cases'][case['id']] = dict(spec=case, **record) + write_json(directories[i] / 'results.json', reports[i]) + for report, directory in zip(reports, directories): + report['complete'] = all(c['status'] == 'ok' for c in report['cases'].values()) + report['finished_utc'] = datetime.datetime.now(datetime.timezone.utc).isoformat() + write_json(directory / 'results.json', report) + if len(reports) == 2: + (output / 'comparison.md').write_text(render_comparison(*reports)) + return 0 if all(r['complete'] for r in reports) else 1 + + +def render_comparison(base, head): + errors = comparison_errors(base, head) + lines = [f'# {base.get("benchmark_id", "GPU benchmark")}', '', + f'Base: `{base["source"]["commit"]}`; candidate: `{head["source"]["commit"]}`.', ''] + if errors: + return '\n'.join(lines + ['**Not comparable:**'] + [f'- {e}' for e in errors]) + '\n' + env = base['environment'] + gpu = env['gpu'] + capability = '.'.join(str(part) for part in gpu['compute_capability']) + def cuda_version(value): + return f'{value // 1000}.{value % 1000 // 10}' + lines += [f'**GPU count: {env.get("gpu_count", "not recorded")}**; versions and cases execute sequentially.', + f'GPU: **{gpu["name"]}**; physical VRAM: **{gpu.get("physical_vram_bytes", gpu["vram_bytes"]) / 2**30:.1f} GiB**; CUDA-visible: **{gpu["vram_bytes"] / 2**30:.1f} GiB**; compute capability: **{capability}**.', + f'UUID: `{gpu["uuid"]}`; PCI bus: `{gpu.get("pci_bus_id", "not recorded")}`.', + f'SMs: {gpu.get("multiprocessors", "not recorded")}; memory bus: {gpu.get("memory_bus_width_bits", "not recorded")} bits; NVIDIA driver: {gpu.get("driver_version", "not recorded")}.', + f'Power limit: {gpu.get("power_limit_watts", "not recorded")} W; max SM clock: {gpu.get("max_sm_clock_mhz", "not recorded")} MHz.', + f'CUDA driver API: {cuda_version(env["cuda_driver"])}; CUDA runtime: {cuda_version(env["cuda_runtime"])}.', '', + 'K is the full target dimension; k is the sketch width used for structure search.', + '`proj`: k projected components; `rand`/`filter`: k draws with replacement (at most k distinct outputs);', + '`topk`: k distinct outputs. Full GradientBoosting uses all K outputs. Predictions always have K outputs.', '', + 'Times per case are warmed wall-clock medians. GPU memory is peak live CuPy allocations,', + 'including the private fit pool; CUDA context, pool cache and external library allocations are excluded.', + 'Quality is a holdout metric (lower is better). These are measured changes, not statistical verdicts.', '', + '| Case | Outputs K | Sketch k | Fit s (base → PR) | Δ fit | Predict s (base → PR) | Δ predict | Fit MiB (base → PR) | Δ memory | Quality (base → PR) |', + '|---|---:|---|---:|---:|---:|---:|---:|---:|---:|'] + notes = [] + if base.get('suite', {}).get('dataset', {}).get('name') == 'Delicious': + # Cross-validation folds are different datasets, not timing repetitions. + table_start = next(i for i, line in enumerate(lines) if line.startswith('| Case |')) + lines[table_start:table_start] = cv_summary(base, head) + for case_id in base['case_ids']: + a, b = base['cases'][case_id], head['cases'][case_id] + def pair(section, key, scale=1): + x, y = a[section][key], b[section][key] + if isinstance(x, dict): + x, y = x['median'], y['median'] + delta = f'{(y / x - 1) * 100:+.1f}%' if x else 'n/a' + return f'{x / scale:.4f} → {y / scale:.4f}', delta + fit, df = pair('timing', 'fit_seconds') + pred, dp = pair('timing', 'predict_seconds') + mem, dm = pair('memory', 'fit_cupy_peak_bytes', 2**20) + qa = a['timing']['quality']['median'] + qb = b['timing']['quality']['median'] + spec = a['spec'] + sketch = spec.get('params', {}).get('sketch_method') + sketch_width = f'{spec["params"]["sketch_outputs"]} ({sketch})' if sketch else f'all {spec["outputs"]}' + lines.append(f'| {case_id} | {spec["outputs"]} | {sketch_width} | {fit} | {df} | {pred} | {dp} | {mem} | {dm} | {qa:.6g} → {qb:.6g} |') + ra = [m['retained_trees'] for m in a['timing']['measurements']] + rb = [m['retained_trees'] for m in b['timing']['measurements']] + if len(set(ra + rb)) > 1: + notes.append(f'Retained tree counts differ for `{case_id}`: {ra} vs {rb}; interpret prediction timing with care.') + if 'fold' in spec: + ta = [m['trained_iterations'] for m in a['timing']['measurements']] + tb = [m['trained_iterations'] for m in b['timing']['measurements']] + notes.append(f'`{case_id}`: executed {ta} → {tb}; retained {ra} → {rb}.') + return '\n'.join(lines + [''] + notes) + '\n' + + +def cv_summary(base, head): + import numpy as np + + lines = ['**Delicious: official first split; five shuffled train folds (seed 42).**', + 'All 983 labels and 500 features; each fold: 10336 train / 2584 validation / 3185 test.', + 'Five-fold means ± sample standard deviations (ddof=1), as in the paper; folds are not timing repeats.', + 'Each memory peak is from a separate full fit; report the maximum across folds.', + 'FilterSketch is an additional library coverage case, outside the paper.', '', + '| Method | K | k | Mean fit s (base → PR) | Δ fit | Mean predict s (base → PR) | Max fit MiB (base → PR) | Mean test log loss (base → PR) |', + '|---|---:|---:|---|---:|---|---|---|'] + groups = list(dict.fromkeys(base['cases'][c]['spec']['group'] for c in base['case_ids'])) + for group in groups: + ids = [c for c in base['case_ids'] if base['cases'][c]['spec']['group'] == group] + if len(ids) != 5: + lines.append(f'Incomplete CV group `{group}`: {len(ids)}/5 folds; no five-fold aggregate.') + continue + def values(report, section, key): + return [report['cases'][c][section][key] for c in ids] + def mean_pair(key): + pairs = [] + means = [] + for report in [base, head]: + data = [v['median'] for v in values(report, 'timing', key)] + means.append(float(np.mean(data))) + pairs.append(f'{means[-1]:.6g} ± {np.std(data, ddof=1):.3g}') + return ' → '.join(pairs), means + fit, means = mean_pair('fit_seconds') + predict, _ = mean_pair('predict_seconds') + score, _ = mean_pair('quality') + peaks = [max(values(r, 'memory', 'fit_cupy_peak_bytes')) / 2**20 for r in [base, head]] + spec = base['cases'][ids[0]]['spec'] + k = spec['params'].get('sketch_outputs', 'all') + lines.append(f'| {group} | 983 | {k} | {fit} | {(means[1]/means[0]-1)*100:+.1f}% | {predict} | ' + f'{peaks[0]:.2f} → {peaks[1]:.2f} | {score} |') + return lines + ['', 'Per-fold measurements:', ''] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + command = commands.add_parser('run', help='Run one or more clean source checkouts') + command.add_argument('--source', action='append', required=True, help='Git checkout; repeat to compare versions') + command.add_argument('--suite', type=Path, default=DEFAULT_SUITE, + help='Frozen suite manifest (default: Delicious paper protocol, k=5)') + command.add_argument('--output', required=True) + command.add_argument('--device', required=True, help='One physical GPU index or UUID') + command.add_argument('--python', default=sys.executable, help='Interpreter containing the pinned dependencies') + command.add_argument('--repeats', type=int, default=None, help='Timing repeats per fold; default from suite') + command.add_argument('--case', action='append', help='Diagnostic subset; omit for the full suite') + command.add_argument('--timeout', type=int, default=None, help='Seconds per worker; default from suite') + command.add_argument('--resume', action='store_true', help='Resume matching reports, skipping successful cases') + command.add_argument('--data-dir', type=Path, default=ROOT.parent / '.benchmark-data') + command = commands.add_parser('prepare-data', help='Download and verify the pinned Delicious dataset') + command.add_argument('--data-dir', type=Path, default=ROOT.parent / '.benchmark-data') + command.add_argument('--archive', type=Path, help='Use an existing original Delicious.zip without network') + command = commands.add_parser('compare', help='Compare two archived results.json files') + command.add_argument('base') + command.add_argument('candidate') + command.add_argument('--output', required=True) + command = commands.add_parser('_worker', help=argparse.SUPPRESS) + command.add_argument('job') + command.add_argument('result') + args = parser.parse_args() + if args.command == 'prepare-data': + if __package__: + from .delicious import prepare_data + else: + from delicious import prepare_data + destination, checksum = prepare_data(args.data_dir, args.archive) + print(f'{destination}\nSHA-256 (arrays and ordered splits): {checksum}') + return 0 + if args.command == '_worker': + try: + result = worker(json.loads(Path(args.job).read_text())) + write_json(args.result, result) + return 0 if result['status'] == 'ok' else 1 + except Exception: + write_json(args.result, {'status': 'error', 'error': traceback.format_exc()}) + traceback.print_exc() + return 1 + if args.command == 'compare': + base, head = [json.loads(Path(p).read_text()) for p in [args.base, args.candidate]] + Path(args.output).write_text(render_comparison(base, head)) + return 2 if comparison_errors(base, head) else 0 + return run(args) + + +if __name__ == '__main__': + try: + sys.exit(main()) + except (ValueError, RuntimeError, subprocess.SubprocessError) as exc: + print(str(exc), file=sys.stderr) + sys.exit(1) diff --git a/benchmarks/suite-delicious-paper-v1-k5.json b/benchmarks/suite-delicious-paper-v1-k5.json new file mode 100644 index 0000000..4e361eb --- /dev/null +++ b/benchmarks/suite-delicious-paper-v1-k5.json @@ -0,0 +1,522 @@ +{ + "benchmark_id": "pyboost-delicious-paper-v1-k5", + "data_seed": 42, + "model_seed": 42, + "cpu_threads": 8, + "warmups": 1, + "repeats": 1, + "sketch_params": { + "proj": { + "norm": true + }, + "rand": { + "smooth": 0.1, + "replace": true + }, + "filter": { + "sample": true, + "smooth": 0.1, + "ntrees": 1 + }, + "topk": {} + }, + "common_params": { + "ntrees": 20000, + "max_depth": 12, + "max_bin": 256, + "lr": 0.015, + "es": 500, + "verbose": 1000000, + "min_data_in_leaf": 76, + "colsample": 1.0, + "subsample": 0.623613094287341, + "use_hess": false, + "lambda_l2": 13.057198025783617, + "gd_steps": 1, + "min_gain_to_split": 0, + "quantization": "Quantile", + "quant_sample": 2000000, + "min_data_in_bin": 3 + }, + "cases": [ + { + "id": "delicious-full-fold0", + "dataset": "delicious", + "fold": 0, + "group": "delicious-full", + "task": "multilabel", + "model": "GradientBoosting", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": {} + }, + { + "id": "delicious-full-fold1", + "dataset": "delicious", + "fold": 1, + "group": "delicious-full", + "task": "multilabel", + "model": "GradientBoosting", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": {} + }, + { + "id": "delicious-full-fold2", + "dataset": "delicious", + "fold": 2, + "group": "delicious-full", + "task": "multilabel", + "model": "GradientBoosting", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": {} + }, + { + "id": "delicious-full-fold3", + "dataset": "delicious", + "fold": 3, + "group": "delicious-full", + "task": "multilabel", + "model": "GradientBoosting", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": {} + }, + { + "id": "delicious-full-fold4", + "dataset": "delicious", + "fold": 4, + "group": "delicious-full", + "task": "multilabel", + "model": "GradientBoosting", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": {} + }, + { + "id": "delicious-topk-fold0", + "dataset": "delicious", + "fold": 0, + "group": "delicious-topk", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "topk", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-topk-fold1", + "dataset": "delicious", + "fold": 1, + "group": "delicious-topk", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "topk", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-topk-fold2", + "dataset": "delicious", + "fold": 2, + "group": "delicious-topk", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "topk", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-topk-fold3", + "dataset": "delicious", + "fold": 3, + "group": "delicious-topk", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "topk", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-topk-fold4", + "dataset": "delicious", + "fold": 4, + "group": "delicious-topk", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "topk", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-rand-fold0", + "dataset": "delicious", + "fold": 0, + "group": "delicious-rand", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "rand", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-rand-fold1", + "dataset": "delicious", + "fold": 1, + "group": "delicious-rand", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "rand", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-rand-fold2", + "dataset": "delicious", + "fold": 2, + "group": "delicious-rand", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "rand", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-rand-fold3", + "dataset": "delicious", + "fold": 3, + "group": "delicious-rand", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "rand", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-rand-fold4", + "dataset": "delicious", + "fold": 4, + "group": "delicious-rand", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "rand", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-proj-fold0", + "dataset": "delicious", + "fold": 0, + "group": "delicious-proj", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-proj-fold1", + "dataset": "delicious", + "fold": 1, + "group": "delicious-proj", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-proj-fold2", + "dataset": "delicious", + "fold": 2, + "group": "delicious-proj", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-proj-fold3", + "dataset": "delicious", + "fold": 3, + "group": "delicious-proj", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-proj-fold4", + "dataset": "delicious", + "fold": 4, + "group": "delicious-proj", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-filter-fold0", + "dataset": "delicious", + "fold": 0, + "group": "delicious-filter", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "filter", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-filter-fold1", + "dataset": "delicious", + "fold": 1, + "group": "delicious-filter", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "filter", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-filter-fold2", + "dataset": "delicious", + "fold": 2, + "group": "delicious-filter", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "filter", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-filter-fold3", + "dataset": "delicious", + "fold": 3, + "group": "delicious-filter", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "filter", + "sketch_outputs": 5 + } + }, + { + "id": "delicious-filter-fold4", + "dataset": "delicious", + "fold": 4, + "group": "delicious-filter", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 10336, + "valid": 2584, + "test": 3185, + "features": 500, + "outputs": 983, + "validation_sets": 1, + "params": { + "sketch_method": "filter", + "sketch_outputs": 5 + } + } + ], + "warmup_iterations": 32, + "timeout": 259200, + "dataset": { + "name": "Delicious", + "rows": 16105, + "features": 500, + "outputs": 983, + "archive_sha256": "9a4e79e9bd1835225e59eef09ba0ee29c5430f785045a9245943511836542806", + "normalized_sha256": "a53ca49cdce00c38f90225994ba01e6b352e9445d121df35fcd177a1cbd5c676", + "source": "https://arxiv.org/html/2211.12858v1", + "reference_commit": "e479fde7a0dd5f624fd9efd654e2d192a12d325d", + "official_split_column": 0, + "cv": { + "folds": 5, + "shuffle": true, + "random_state": 42 + }, + "metric_clip": 1e-07 + } +} diff --git a/benchmarks/suite-v1-k5.json b/benchmarks/suite-v1-k5.json new file mode 100644 index 0000000..4b9cf1c --- /dev/null +++ b/benchmarks/suite-v1-k5.json @@ -0,0 +1,231 @@ +{ + "benchmark_id": "pyboost-gpu-v1-k5", + "data_seed": 20261030, + "model_seed": 42, + "cpu_threads": 4, + "warmups": 1, + "repeats": 5, + "sketch_params": { + "proj": { + "norm": true + }, + "rand": { + "smooth": 0.1, + "replace": true + }, + "filter": { + "sample": true, + "smooth": 0.1, + "ntrees": 1 + }, + "topk": {} + }, + "common_params": { + "ntrees": 32, + "max_depth": 4, + "max_bin": 64, + "lr": 0.1, + "es": 0, + "verbose": 1000000, + "min_data_in_leaf": 10, + "colsample": 1.0, + "subsample": 1.0, + "use_hess": true, + "lambda_l2": 1.0, + "gd_steps": 1, + "min_gain_to_split": 0, + "quantization": "Quantile", + "quant_sample": 2000000, + "min_data_in_bin": 3 + }, + "cases": [ + { + "id": "mse-single", + "task": "regression", + "model": "GradientBoosting", + "loss": "mse", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 1, + "validation_sets": 1, + "params": {} + }, + { + "id": "mse-full-128", + "task": "regression", + "model": "GradientBoosting", + "loss": "mse", + "train": 8192, + "valid": 2048, + "test": 2048, + "features": 32, + "outputs": 128, + "validation_sets": 1, + "params": {} + }, + { + "id": "mse-proj-32", + "task": "regression", + "model": "SketchBoost", + "loss": "mse", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 32, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "mse-proj-128", + "task": "regression", + "model": "SketchBoost", + "loss": "mse", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 128, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "mse-proj-1024", + "task": "regression", + "model": "SketchBoost", + "loss": "mse", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 1024, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "mse-proj-10000", + "task": "regression", + "model": "SketchBoost", + "loss": "mse", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 10000, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "bce-rand-1024", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 1024, + "validation_sets": 1, + "params": { + "sketch_method": "rand", + "sketch_outputs": 5 + } + }, + { + "id": "bce-filter-1024", + "task": "multilabel", + "model": "SketchBoost", + "loss": "multilabel", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 1024, + "validation_sets": 1, + "params": { + "sketch_method": "filter", + "sketch_outputs": 5 + } + }, + { + "id": "mse-topk-10000", + "task": "regression", + "model": "SketchBoost", + "loss": "mse", + "train": 4096, + "valid": 1024, + "test": 1024, + "features": 32, + "outputs": 10000, + "validation_sets": 1, + "params": { + "sketch_method": "topk", + "sketch_outputs": 5 + } + }, + { + "id": "ce-proj-512", + "task": "multiclass", + "model": "SketchBoost", + "loss": "crossentropy", + "train": 16384, + "valid": 4096, + "test": 4096, + "features": 32, + "outputs": 512, + "validation_sets": 1, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "validation-proj-1024", + "task": "regression", + "model": "SketchBoost", + "loss": "mse", + "train": 8192, + "valid": 32768, + "test": 4096, + "features": 32, + "outputs": 1024, + "validation_sets": 2, + "params": { + "sketch_method": "proj", + "sketch_outputs": 5 + } + }, + { + "id": "histogram-topk-128", + "task": "regression", + "model": "SketchBoost", + "loss": "mse", + "train": 8192, + "valid": 2048, + "test": 2048, + "features": 128, + "outputs": 128, + "validation_sets": 1, + "params": { + "sketch_method": "topk", + "sketch_outputs": 5, + "max_depth": 8, + "max_bin": 256 + } + } + ] +} diff --git a/benchmarks/suite-v1.json b/benchmarks/suite-v1.json new file mode 100644 index 0000000..ac87722 --- /dev/null +++ b/benchmarks/suite-v1.json @@ -0,0 +1,35 @@ +{ + "benchmark_id": "pyboost-gpu-v1", + "data_seed": 20261030, + "model_seed": 42, + "cpu_threads": 4, + "warmups": 1, + "repeats": 5, + "sketch_params": { + "proj": {"norm": true}, + "rand": {"smooth": 0.1, "replace": true}, + "filter": {"sample": true, "smooth": 0.1, "ntrees": 1}, + "topk": {} + }, + "common_params": { + "ntrees": 32, "max_depth": 4, "max_bin": 64, "lr": 0.1, + "es": 0, "verbose": 1000000, "min_data_in_leaf": 10, + "colsample": 1.0, "subsample": 1.0, "use_hess": true, + "lambda_l2": 1.0, "gd_steps": 1, "min_gain_to_split": 0, + "quantization": "Quantile", "quant_sample": 2000000, "min_data_in_bin": 3 + }, + "cases": [ + {"id": "mse-single", "task": "regression", "model": "GradientBoosting", "loss": "mse", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 1, "validation_sets": 1, "params": {}}, + {"id": "mse-full-128", "task": "regression", "model": "GradientBoosting", "loss": "mse", "train": 8192, "valid": 2048, "test": 2048, "features": 32, "outputs": 128, "validation_sets": 1, "params": {}}, + {"id": "mse-proj-32", "task": "regression", "model": "SketchBoost", "loss": "mse", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 32, "validation_sets": 1, "params": {"sketch_method": "proj", "sketch_outputs": 8}}, + {"id": "mse-proj-128", "task": "regression", "model": "SketchBoost", "loss": "mse", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 128, "validation_sets": 1, "params": {"sketch_method": "proj", "sketch_outputs": 8}}, + {"id": "mse-proj-1024", "task": "regression", "model": "SketchBoost", "loss": "mse", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 1024, "validation_sets": 1, "params": {"sketch_method": "proj", "sketch_outputs": 8}}, + {"id": "mse-proj-10000", "task": "regression", "model": "SketchBoost", "loss": "mse", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 10000, "validation_sets": 1, "params": {"sketch_method": "proj", "sketch_outputs": 8}}, + {"id": "bce-rand-1024", "task": "multilabel", "model": "SketchBoost", "loss": "multilabel", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 1024, "validation_sets": 1, "params": {"sketch_method": "rand", "sketch_outputs": 8}}, + {"id": "bce-filter-1024", "task": "multilabel", "model": "SketchBoost", "loss": "multilabel", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 1024, "validation_sets": 1, "params": {"sketch_method": "filter", "sketch_outputs": 8}}, + {"id": "mse-topk-10000", "task": "regression", "model": "SketchBoost", "loss": "mse", "train": 4096, "valid": 1024, "test": 1024, "features": 32, "outputs": 10000, "validation_sets": 1, "params": {"sketch_method": "topk", "sketch_outputs": 8}}, + {"id": "ce-proj-512", "task": "multiclass", "model": "SketchBoost", "loss": "crossentropy", "train": 16384, "valid": 4096, "test": 4096, "features": 32, "outputs": 512, "validation_sets": 1, "params": {"sketch_method": "proj", "sketch_outputs": 8}}, + {"id": "validation-proj-1024", "task": "regression", "model": "SketchBoost", "loss": "mse", "train": 8192, "valid": 32768, "test": 4096, "features": 32, "outputs": 1024, "validation_sets": 2, "params": {"sketch_method": "proj", "sketch_outputs": 8}}, + {"id": "histogram-topk-128", "task": "regression", "model": "SketchBoost", "loss": "mse", "train": 8192, "valid": 2048, "test": 2048, "features": 128, "outputs": 128, "validation_sets": 1, "params": {"sketch_method": "topk", "sketch_outputs": 8, "max_depth": 8, "max_bin": 256}} + ] +} diff --git a/tests/test_benchmark_protocol.py b/tests/test_benchmark_protocol.py new file mode 100644 index 0000000..bf7406a --- /dev/null +++ b/tests/test_benchmark_protocol.py @@ -0,0 +1,246 @@ +"""The result protocol must reject misleading cross-version comparisons.""" + +from types import SimpleNamespace +import importlib.util +from pathlib import Path +import sys + +import numpy as np +import pytest + +# The CPU wheel workflow runs tests outside the source root. Load only the +# benchmark package, without putting the library checkout ahead of the wheel. +_spec = importlib.util.spec_from_file_location( + 'pyboost_benchmark_suite', Path(__file__).resolve().parents[1] / 'benchmarks' / '__init__.py') +_package = importlib.util.module_from_spec(_spec) +sys.modules[_spec.name] = _package +_spec.loader.exec_module(_package) + +from pyboost_benchmark_suite.gpu import memory_tracker +from pyboost_benchmark_suite.protocol import ROOT, comparison_errors, implementation_hash, load_suite, make_data, quality +from pyboost_benchmark_suite.run import environment, render_comparison +from pyboost_benchmark_suite import run as runner + + +def small_case(task='regression'): + return dict(id='test', train=64, valid=16, test=16, features=32, + outputs=8, validation_sets=2, task=task) + + +@pytest.mark.parametrize('task', ['regression', 'multilabel', 'multiclass']) +def test_datasets_are_reproducible_and_separated(task): + case = small_case(task) + a, b = make_data(case, 42), make_data(case, 42) + assert a[-1] == b[-1] + assert a[-1] != make_data(case, 43)[-1] + np.testing.assert_array_equal(a[0], b[0]) + np.testing.assert_array_equal(a[1], b[1]) + assert a[0].shape == (64, 32) + assert [len(v['y']) for v in a[2]] == [8, 8] + assert not np.shares_memory(a[0], a[3]) + if task == 'multiclass': + np.testing.assert_array_equal(np.unique(a[1]), np.arange(8)) + elif task == 'multilabel': + assert set(np.unique(a[1])) == {0, 1} + else: + assert quality(task, a[1], a[1])['value'] == 0 + + +def test_same_multilabel_data_for_different_sketch_methods(): + suite = load_suite(ROOT / 'suite-v1-k5.json') + rand, filt = [next(c for c in suite['cases'] if c['id'] == name) + for name in ['bce-rand-1024', 'bce-filter-1024']] + for case in [rand, filt]: + case.update(train=64, valid=16, test=16, outputs=8) + assert make_data(rand, suite['data_seed'])[-1] == make_data(filt, suite['data_seed'])[-1] + + +def test_k5_profile_changes_only_sketch_width_and_identity(): + from copy import deepcopy + original = load_suite(ROOT / 'suite-v1.json') + k5 = load_suite(ROOT / 'suite-v1-k5.json') + assert k5['benchmark_id'] == 'pyboost-gpu-v1-k5' + expected = deepcopy(original) + expected['benchmark_id'] = k5['benchmark_id'] + for case in expected['cases']: + if case['model'] == 'SketchBoost': + case['params']['sketch_outputs'] = 5 + assert k5 == expected + assert implementation_hash(k5) != implementation_hash(original) + assert all(c['params']['sketch_outputs'] == 8 for c in original['cases'] + if c['model'] == 'SketchBoost') + before, after = small_case(), small_case() + before['params'] = dict(sketch_outputs=8) + after['params'] = dict(sketch_outputs=5) + assert make_data(before, 42)[-1] == make_data(after, 42)[-1] + + +def test_cli_defaults_to_k5(monkeypatch, tmp_path): + captured = {} + def capture(args): + captured['suite'] = load_suite(args.suite) + return 0 + monkeypatch.setattr(runner, 'run', capture) + monkeypatch.setattr(sys, 'argv', ['run.py', 'run', '--source', 'checkout', + '--device', '1', '--output', str(tmp_path)]) + assert runner.main() == 0 + assert captured['suite']['benchmark_id'] == 'pyboost-delicious-paper-v1-k5' + assert all(c['params']['sketch_outputs'] == 5 for c in captured['suite']['cases'] + if c['model'] == 'SketchBoost') + + +def test_holdout_metrics_and_invalid_predictions(): + assert quality('multilabel', np.array([[0, 1]], dtype=np.float32), + np.array([[.5, .5]], dtype=np.float32))['value'] == pytest.approx(np.log(2)) + assert quality('multiclass', np.array([1]), np.array([[.25, .75]]))['value'] == pytest.approx(-np.log(.75)) + with pytest.raises(ValueError, match='Non-finite'): + quality('regression', np.zeros((1, 1)), np.array([[np.nan]])) + with pytest.raises(ValueError, match='output count'): + quality('regression', np.zeros((1, 2)), np.zeros((1, 1))) + + +def report(): + timing = dict(fit_seconds={'median': 1.}, predict_seconds={'median': .1}, + quality={'median': .2}, measurements=[{'retained_trees': 32}]) + return dict(schema_version=1, benchmark_id='pyboost-gpu-v1', implementation_hash='abc', + suite_hash='def', case_ids=['test'], repeats=5, warmups=1, complete=True, + environment={'gpu_count': 1, 'gpu': {'name': 'NVIDIA A100 80GB PCIe', + 'uuid': 'GPU-test', 'vram_bytes': 80 * 2**30, 'compute_capability': [8, 0], + 'physical_vram_bytes': 80 * 2**30, 'power_limit_watts': '300', + 'max_sm_clock_mhz': '1410', + 'pci_bus_id': '00000000:14:00.0', 'multiprocessors': 108, + 'memory_bus_width_bits': 5120, 'driver_version': '550.163.01'}, + 'cuda_driver': 12040, 'cuda_runtime': 12090, 'packages': {'numpy': '2'}}, + source={'commit': 'abc'}, cases={'test': dict(status='ok', data_hash='123', + spec={'outputs': 1024}, timing=timing, memory={'fit_cupy_peak_bytes': 2**20})}) + + +@pytest.mark.parametrize('key,value', [ + ('implementation_hash', 'changed'), ('suite_hash', 'changed'), + ('environment', {'gpu': 'T4'}), ('repeats', 1), ('complete', False), +]) +def test_incompatible_measurements_are_not_compared(key, value): + base, candidate = report(), report() + candidate[key] = value + assert comparison_errors(base, candidate) + text = render_comparison(base, candidate) + assert 'Not comparable' in text + assert '| Case |' not in text + + +@pytest.mark.parametrize('change', ['dataset', 'error', 'missing_case', 'spec']) +def test_incomplete_or_different_workloads_are_rejected(change): + base, candidate = report(), report() + if change == 'dataset': + candidate['cases']['test']['data_hash'] = 'different' + elif change == 'error': + candidate['cases']['test']['status'] = 'error' + elif change == 'missing_case': + candidate['cases'].clear() + else: + candidate['cases']['test']['spec']['outputs'] = 1 + assert comparison_errors(base, candidate) + + +def test_comparison_preserves_direction_and_discloses_tree_count_changes(): + base, candidate = report(), report() + candidate['source']['commit'] = 'new' + candidate['cases']['test']['timing']['fit_seconds']['median'] = 1.5 + candidate['cases']['test']['memory']['fit_cupy_peak_bytes'] = 2**19 + candidate['cases']['test']['timing']['measurements'][0]['retained_trees'] = 31 + assert comparison_errors(base, candidate) == [] + text = render_comparison(base, candidate) + assert '+50.0%' in text + assert '-50.0%' in text + assert 'Retained tree counts differ' in text + + +def test_report_displays_single_gpu_characteristics(): + text = render_comparison(report(), report()) + for value in ['GPU count: 1', 'NVIDIA A100 80GB PCIe', '80.0 GiB', + 'GPU-test', '8.0', '108', '5120', '550.163.01', '12.4', '12.9', + '300 W', '1410 MHz']: + assert value in text + + +@pytest.mark.parametrize('method', ['proj', 'rand', 'filter', 'topk']) +def test_report_distinguishes_target_dimension_from_sketch_width(method): + base, candidate = report(), report() + for result in [base, candidate]: + result['cases']['test']['spec'] = dict(outputs=10000, params=dict( + sketch_method=method, sketch_outputs=8)) + text = render_comparison(base, candidate) + assert '| test | 10000 | 8 (' + method + ') |' in text + assert 'at most k distinct outputs' in text + assert 'Predictions always have K outputs' in text + + +def test_report_shows_full_output_scoring_without_sketch(): + text = render_comparison(report(), report()) + assert '| test | 1024 | all 1024 |' in text + + +@pytest.mark.parametrize('count', [0, 2]) +def test_preflight_rejects_any_visible_gpu_count_other_than_one(count): + cp = SimpleNamespace(cuda=SimpleNamespace(runtime=SimpleNamespace(getDeviceCount=lambda: count))) + with pytest.raises(RuntimeError, match='Exactly one visible CUDA device'): + environment(cp, np, None, '0') + + +def test_cli_rejects_multiple_gpu_ids_before_starting_workers(monkeypatch, tmp_path): + monkeypatch.setattr(runner, 'source_info', lambda path: dict(commit='abc', path=str(path))) + monkeypatch.setattr(runner, 'invoke', lambda *args: pytest.fail('Worker must not start')) + output = tmp_path / 'results' + args = SimpleNamespace(source=['checkout'], suite=None, case=None, repeats=5, timeout=900, + device='0,1', output=str(output)) + with pytest.raises(ValueError, match='exactly one GPU'): + runner.run(args) + assert not output.exists() + + +def test_gpu_index_is_resolved_to_physical_uuid(monkeypatch): + commands = [] + def query(command, **kwargs): + commands.append(command) + return 'GPU-selected\n' + monkeypatch.setattr(runner.subprocess, 'check_output', query) + assert runner.resolve_gpu('1') == 'GPU-selected' + assert commands == [['nvidia-smi', '-i', '1', '--query-gpu=uuid', '--format=csv,noheader']] + + +@pytest.mark.parametrize('response', ['', 'GPU-a\nGPU-b\n', 'MIG-instance\n']) +def test_gpu_selector_must_resolve_to_one_physical_card(monkeypatch, response): + monkeypatch.setattr(runner.subprocess, 'check_output', lambda *args, **kwargs: response) + with pytest.raises(ValueError, match='exactly one physical GPU'): + runner.resolve_gpu('selector') + + +def test_live_memory_tracking_handles_reuse_and_preexisting_allocations(): + tracker = memory_tracker(SimpleNamespace(cuda=SimpleNamespace(MemoryHook=object))) + tracker.free_postprocess(0, 512, 1, 999) # Allocated before the tracked scope. + tracker.malloc_postprocess(0, 100, 512, 1, 1) + tracker.malloc_postprocess(0, 800, 1024, 2, 2) + assert tracker.current == tracker.peak == 1536 + tracker.free_postprocess(0, 512, 1, 1) + tracker.malloc_postprocess(0, 100, 512, 1, 3) # Reused pool block. + assert tracker.current == tracker.peak == 1536 + tracker.free_postprocess(0, 1024, 2, 2) + tracker.reset_peak() + assert tracker.current == tracker.peak == 512 + tracker.free_postprocess(0, 512, 1, 3) + assert tracker.current == 0 + + +def test_real_hook_sees_private_cupy_pool(): + cp = pytest.importorskip('cupy') + if not cp.is_available(): + pytest.skip('Requires a CUDA device') + pool = cp.cuda.MemoryPool() + with memory_tracker(cp) as tracker: + with cp.cuda.using_allocator(pool.malloc): + a = cp.empty(1024, dtype=cp.float32) + b = cp.empty(1024, dtype=cp.float64) + assert tracker.current == tracker.peak == a.nbytes + b.nbytes + del a, b + assert tracker.current == 0 + pool.free_all_blocks() diff --git a/tests/test_delicious_benchmark.py b/tests/test_delicious_benchmark.py new file mode 100644 index 0000000..17466fa --- /dev/null +++ b/tests/test_delicious_benchmark.py @@ -0,0 +1,112 @@ +"""Dataset integrity, official split isolation and five-fold report aggregation.""" + +from copy import deepcopy +import json +from types import SimpleNamespace + +import numpy as np +import pytest + +from test_benchmark_protocol import report +from pyboost_benchmark_suite import delicious +from pyboost_benchmark_suite.protocol import load_suite +from pyboost_benchmark_suite.run import cv_summary +from pyboost_benchmark_suite import run as runner + + +def test_paper_profile_has_all_folds_and_outputs(): + suite = load_suite() + assert suite['benchmark_id'] == 'pyboost-delicious-paper-v1-k5' + assert len(suite['cases']) == 25 + for method in ['full', 'topk', 'rand', 'proj', 'filter']: + cases = [c for c in suite['cases'] if c['group'] == f'delicious-{method}'] + assert {c['fold'] for c in cases} == set(range(5)) + assert all(c['outputs'] == 983 for c in cases) + params = suite['common_params'] + assert (params['ntrees'], params['es'], params['lr'], params['max_depth']) == (20000, 500, .015, 12) + assert params['use_hess'] is False + assert suite['repeats'] == 1 # Different folds, rather than five repeats per fold. + + +def test_corrupted_archive_rejected_before_parsing(tmp_path): + archive = tmp_path / 'Delicious.zip' + archive.write_bytes(b'not the official archive') + with pytest.raises(ValueError, match='archive SHA-256 mismatch'): + delicious.prepare_data(tmp_path, archive) + assert not (tmp_path / 'delicious.npz').exists() + + +def test_official_test_set_never_enters_folds_and_validation_covers_train(monkeypatch, tmp_path): + # Tiny checked arrays avoid downloading a dataset during unit tests. + X = np.arange(60, dtype=np.float32).reshape(20, 3) + y = np.zeros((20, 2), dtype=np.float32) + train, test = np.arange(15), np.arange(15, 20) + np.savez(tmp_path / 'delicious.npz', X=X, y=y, train=train, test=test) + monkeypatch.setattr(delicious, 'NORMALIZED_SHA256', delicious.array_hash(X, y, train, test)) + validations = [] + hashes = [] + for fold in range(5): + case = dict(fold=fold, train=12, valid=3, test=5) + a, b, valid, tx, ty, digest = delicious.load_fold(case, 42, tmp_path) + np.testing.assert_array_equal(tx, X[test]) + np.testing.assert_array_equal(ty, y[test]) + fit_ids = set(a[:, 0]) + valid_ids = set(valid[0]['X'][:, 0]) + assert fit_ids.isdisjoint(valid_ids) + assert (fit_ids | valid_ids) == set(X[train, 0]) + validations.extend(valid_ids) + hashes.append(digest) + assert digest == delicious.load_fold(case, 42, tmp_path)[-1] + assert sorted(validations) == sorted(X[train, 0]) + assert len(set(hashes)) == 5 + y[0, 0] = 1 + np.savez(tmp_path / 'delicious.npz', X=X, y=y, train=train, test=test) + with pytest.raises(ValueError, match='checksum mismatch'): + delicious.load_fold(case, 42, tmp_path) + + +def test_cv_uses_mean_not_median_and_maximum_memory(): + base = report() + sample = base['cases']['test'] + base['cases'] = {} + base['case_ids'] = [] + for fold, seconds in enumerate([1., 1., 1., 1., 6.]): + key = f'fold{fold}' + value = deepcopy(sample) + value['spec'] = dict(group='delicious-proj', fold=fold, params=dict(sketch_outputs=5)) + value['timing']['fit_seconds']['median'] = seconds + value['memory']['fit_cupy_peak_bytes'] = (fold + 1) * 2**20 + base['cases'][key] = value + base['case_ids'].append(key) + candidate = deepcopy(base) + text = '\n'.join(cv_summary(base, candidate)) + assert '| delicious-proj | 983 | 5 | 2 ± 2.24 → 2 ± 2.24 |' in text + assert '| 5.00 → 5.00 |' in text + base['case_ids'].pop() + assert 'Incomplete CV group' in '\n'.join(cv_summary(base, base)) + + +def test_resume_skips_completed_folds_and_rejects_changed_sources(monkeypatch, tmp_path): + suite = load_suite() + suite.pop('dataset') + suite['cases'] = suite['cases'][:1] + manifest = tmp_path / 'suite.json' + manifest.write_text(json.dumps(suite)) + calls = [] + source = dict(commit='123456789abc', path='checkout', library_hash='original') + monkeypatch.setattr(runner, 'source_info', lambda path: source.copy()) + monkeypatch.setattr(runner, 'resolve_gpu', lambda selector: 'GPU-test') + def invoke(python, path, job, timeout): + calls.append(job['phase']) + return dict(status='ok', environment={'gpu': 'test'}) + monkeypatch.setattr(runner, 'invoke', invoke) + args = SimpleNamespace(suite=manifest, source=['checkout'], repeats=None, + timeout=None, case=None, device='1', python='python', + output=tmp_path / 'out', data_dir=tmp_path, resume=False) + assert runner.run(args) == 0 + args.resume = True + assert runner.run(args) == 0 + assert calls == ['metadata', 'case', 'metadata'] + source['library_hash'] = 'changed' + with pytest.raises(ValueError, match='different source'): + runner.run(args)