diff --git a/py_boost/callbacks/callback.py b/py_boost/callbacks/callback.py index 7817b69..21012dd 100644 --- a/py_boost/callbacks/callback.py +++ b/py_boost/callbacks/callback.py @@ -195,7 +195,7 @@ def after_iteration(self, build_info): build_info['iter_score'] = val_metric - if ((num_iter % self.verbose) == 0) or (num_iter == (self.ntrees - 1)): + if self.verbose > 0 and (((num_iter % self.verbose) == 0) or (num_iter == (self.ntrees - 1))): logger.info(msg) @staticmethod diff --git a/py_boost/gpu/boosting.py b/py_boost/gpu/boosting.py index 0b5d2bb..0c7443a 100644 --- a/py_boost/gpu/boosting.py +++ b/py_boost/gpu/boosting.py @@ -1,5 +1,7 @@ """Gradient boosting builder""" +from numbers import Real + try: import cupy as cp except Exception: @@ -14,6 +16,22 @@ from ..sampling.bagging import BaseSampler +def _get_sampler(value, axis, name): + """Validate a sampling rate or preserve a custom sampler callback.""" + if isinstance(value, Real) and not isinstance(value, bool): + if not 0 < value <= 1: + raise ValueError(f'{name} must be in (0, 1], got {value!r}') + # Keep axis-length arithmetic out of NumPy's low-precision scalar dtypes. + return BaseSampler(float(value), axis=axis) + + hooks = ('before_train', 'before_iteration', 'after_iteration', 'after_train') + if callable(value) and all(callable(getattr(value, hook, None)) for hook in hooks): + return value + + raise TypeError(f'{name} must be a real number in (0, 1] or a callable sampler ' + 'implementing before_train, before_iteration, after_iteration and after_train') + + class GradientBoosting(Ensemble): """Basic Gradient Boosting on depthwise trees""" @@ -59,8 +77,8 @@ def __init__(self, loss, max_depth: int > 0, maximum tree depth. Setting it to large values (>12) may cause OOM for wide datasets min_data_in_leaf: int, minimal leaf size. Note - for some loss fn leaf size is approximated with hessian values to speed up training - colsample: float or Callable, sumsample of columns to construct trees or callable - custom sampling - subsample: float or Callable, sumsample of rows to construct trees or callable - custom sampling + colsample: real number in (0, 1] or sampler callback, fraction of columns to construct trees + subsample: real number in (0, 1] or sampler callback, fraction of rows to construct trees quant_sample: int, subsample to quantize features target_splitter: str or Callable, target splitter, defined multioutput strategy: 'Single', 'OneVsAll' or custom @@ -76,7 +94,7 @@ def __init__(self, loss, es: int, early stopping rounds. If 0, no early stopping seed: int, random state - verbose: int, verbosity freq + verbose: int, logging interval; <=0 disables progress logging callbacks: list of Callback, callbacks to customize training are passed here debug: bool, if debug mode is enabled (it removes ability to use deprecated functions, @@ -132,13 +150,8 @@ def _infer_params(self): self.min_data_in_leaf = self.params['min_data_in_leaf'] self.use_hess = self.params['use_hess'] - self.colsample = self.params['colsample'] - if type(self.params['colsample']) in [float, int]: - self.colsample = BaseSampler(self.params['colsample'], axis=1) - - self.subsample = self.params['subsample'] - if type(self.params['subsample']) in [float, int]: - self.subsample = BaseSampler(self.params['subsample'], axis=0) + self.colsample = _get_sampler(self.params['colsample'], axis=1, name='colsample') + self.subsample = _get_sampler(self.params['subsample'], axis=0, name='subsample') if self.params['target_splitter'] == 'Single': splitter = SingleSplitter() diff --git a/py_boost/gpu/sketch_boost.py b/py_boost/gpu/sketch_boost.py index bf3ad13..15a9a36 100644 --- a/py_boost/gpu/sketch_boost.py +++ b/py_boost/gpu/sketch_boost.py @@ -52,8 +52,8 @@ def __init__(self, loss, max_depth: int > 0, maximum tree depth. Setting it to large values (>12) may cause OOM for wide datasets min_data_in_leaf: int, minimal leaf size. Note - for some loss fn leaf size is approximated with hessian values to speed up training - colsample: float or Callable, sumsample of columns to construct trees or callable - custom sampling - subsample: float or Callable, sumsample of rows to construct trees or callable - custom sampling + colsample: real number in (0, 1] or sampler callback, fraction of columns to construct trees + subsample: real number in (0, 1] or sampler callback, fraction of rows to construct trees quantization: str or Quantizer, method for quantizatrion. One of 'Quantile', 'Uniform', 'Uniquant' or custom implementation @@ -63,7 +63,7 @@ def __init__(self, loss, es: int, early stopping rounds. If 0, no early stopping seed: int, random state - verbose: int, verbosity freq + verbose: int, logging interval; <=0 disables progress logging sketch_outputs: int, number of outputs to keep sketch_method: str, name of the sketching strategy use_hess: bool, use hessians in multioutput training diff --git a/tests/test_sampling_parameters.py b/tests/test_sampling_parameters.py new file mode 100644 index 0000000..e329154 --- /dev/null +++ b/tests/test_sampling_parameters.py @@ -0,0 +1,160 @@ +"""Regression coverage for numeric sampling rates and custom samplers (#27).""" + +import numpy as np +import pytest + +cp = pytest.importorskip('cupy') +if not cp.is_available(): + pytest.skip('Sampling tests require a CUDA device', allow_module_level=True) + +from py_boost import Callback, GradientBoosting, SketchBoost +from py_boost.sampling.bagging import BaseSampler + + +@pytest.fixture +def regression_data(): + rng = np.random.default_rng(42) + X = rng.normal(size=(128, 8)).astype(np.float32) + y = (2 * X[:, 0] - X[:, 1]).astype(np.float32) + return X, y + + +@pytest.mark.parametrize('name', ['colsample', 'subsample']) +@pytest.mark.parametrize('rate', [ + pytest.param(0.5, id='python-float'), + pytest.param(1, id='python-int'), + pytest.param(1.0, id='full-sample-float'), + pytest.param(np.float16(0.5), id='float16'), + pytest.param(np.float32(0.5), id='float32'), + pytest.param(np.float64(0.5), id='float64'), + pytest.param(np.int32(1), id='int32'), + pytest.param(np.int64(1), id='int64'), +]) +def test_numpy_sampling_rates_train_like_python(regression_data, name, rate): + X, y = regression_data + params = dict(loss='mse', ntrees=3, max_depth=2, es=0, verbose=0) + model = GradientBoosting(**params, **{name: rate}).fit(X, y) + reference = GradientBoosting(**params, **{name: float(rate)}).fit(X, y) + + sampler = getattr(model, name) + assert isinstance(sampler, BaseSampler) + assert sampler.axis == (1 if name == 'colsample' else 0) + assert len(model.models) == 3 + actual = model.predict(X) + assert np.isfinite(actual).all() + assert np.std(actual) > 0 + np.testing.assert_allclose(actual, reference.predict(X), rtol=1e-5, atol=1e-6) + + +@pytest.mark.parametrize('name, axis', [('colsample', 1), ('subsample', 0)]) +@pytest.mark.parametrize('length', [65_504, 65_520, 100_000]) +def test_float16_rates_on_large_sampling_axes(name, axis, length): + model = GradientBoosting('mse', **{name: np.float16(0.5)}) + model._infer_params() + sampler = getattr(model, name) + shape = [1, 1] + shape[axis] = length + build_info = {'data': {'train': {'features_gpu': cp.empty(shape, dtype=cp.uint8)}}} + + # NumPy 2 can cast the axis length to float16 before multiplication. + sampler.before_train(build_info) + assert sampler().shape == (length // 2,) + sampler.after_train(build_info) + + +def test_float16_subsample_trains_on_large_dataset(): + X = np.random.default_rng(42).normal(size=(100_000, 2)).astype(np.float32) + y = X[:, 0].copy() + params = dict(loss='mse', ntrees=2, max_depth=2, es=0, verbose=0) + model = GradientBoosting(**params, subsample=np.float16(0.5)).fit(X, y) + reference = GradientBoosting(**params, subsample=0.5).fit(X, y) + + assert len(model.models) == 2 + actual = model.predict(X) + assert np.isfinite(actual).all() + assert np.std(actual) > 0 + np.testing.assert_allclose(actual, reference.predict(X), rtol=1e-5, atol=1e-6) + + +@pytest.mark.parametrize('name', ['colsample', 'subsample']) +@pytest.mark.parametrize('rate', [ + 0, -0.1, 1.1, np.nan, np.inf, -np.inf, + np.float32(0), np.float64(np.nan), np.int64(2), +]) +def test_invalid_sampling_rates_raise_clear_errors(regression_data, name, rate): + X, y = regression_data + model = GradientBoosting('mse', **{name: rate}) + with pytest.raises(ValueError, match=rf'{name}.*\(0, 1\]'): + model.fit(X, y) + + +@pytest.mark.parametrize('name', ['colsample', 'subsample']) +@pytest.mark.parametrize('value', [ + True, False, np.bool_(True), '0.5', None, object(), 0.5j, + np.array(0.5), np.array([0.5]), lambda: None, Callback(), +]) +def test_invalid_sampler_types_raise_clear_errors(regression_data, name, value): + X, y = regression_data + model = GradientBoosting('mse', **{name: value}) + with pytest.raises(TypeError, match=name): + model.fit(X, y) + + +class CustomSampler: + """A sampler implementing the callback protocol without inheriting Callback.""" + + def __init__(self, axis): + self.sampler = BaseSampler(0.5, axis=axis) + self.calls = [] + + def before_train(self, build_info): + self.calls.append('before_train') + self.sampler.before_train(build_info) + + def before_iteration(self, build_info): + self.calls.append('before_iteration') + self.sampler.before_iteration(build_info) + + def __call__(self): + self.calls.append('sample') + return self.sampler() + + def after_iteration(self, build_info): + self.calls.append('after_iteration') + return False + + def after_train(self, build_info): + self.calls.append('after_train') + self.sampler.after_train(build_info) + + +class CallbackSampler(CustomSampler, Callback): + """The same protocol implemented by a Callback subclass.""" + + +@pytest.mark.parametrize('name, axis', [('colsample', 1), ('subsample', 0)]) +@pytest.mark.parametrize('sampler_type', [CustomSampler, CallbackSampler]) +def test_custom_sampler_lifecycle_is_preserved(regression_data, name, axis, sampler_type): + X, y = regression_data + sampler = sampler_type(axis) + params = dict(loss='mse', ntrees=3, max_depth=2, es=0, verbose=0) + model = GradientBoosting(**params, **{name: sampler}).fit(X, y) + reference = GradientBoosting(**params, **{name: 0.5}).fit(X, y) + + assert getattr(model, name) is sampler + assert sampler.calls[0] == 'before_train' + assert sampler.calls[-1] == 'after_train' + assert sampler.calls.count('before_iteration') == 3 + assert sampler.calls.count('after_iteration') == 3 + assert sampler.calls.count('sample') == 3 + np.testing.assert_allclose(model.predict(X), reference.predict(X), rtol=1e-5, atol=1e-6) + + +@pytest.mark.parametrize('model_type', [GradientBoosting, SketchBoost]) +def test_numpy_rates_work_together(regression_data, model_type): + X, y = regression_data + y = np.column_stack([y, -y]) + params = dict(loss='mse', ntrees=3, max_depth=2, es=0, verbose=0) + model = model_type(**params, colsample=np.float32(0.5), subsample=np.float64(0.5)).fit(X, y) + reference = model_type(**params, colsample=0.5, subsample=0.5).fit(X, y) + np.testing.assert_allclose(model.predict(X), reference.predict(X), rtol=1e-5, atol=1e-6) diff --git a/tests/test_training_verbosity.py b/tests/test_training_verbosity.py new file mode 100644 index 0000000..5fa4822 --- /dev/null +++ b/tests/test_training_verbosity.py @@ -0,0 +1,65 @@ +"""Regression coverage for disabled progress logging (#33).""" + +from types import SimpleNamespace + +import numpy as np +import pytest + +cp = pytest.importorskip('cupy') +if not cp.is_available(): + pytest.skip('Training tests require a CUDA device', allow_module_level=True) + +from py_boost import GradientBoosting +from py_boost.callbacks import callback + + +@pytest.mark.parametrize('verbose, expected', [ + (0, []), (-1, []), (1, [0, 1, 2, 3, 4, 5]), (2, [0, 2, 4, 5]), (10, [0, 5]), +]) +def test_iteration_logging_intervals(monkeypatch, verbose, expected): + messages = [] + monkeypatch.setattr(callback.logger, 'info', messages.append) + history = callback.EvalHistory([], verbose=verbose) + history.metric = SimpleNamespace(alias='RMSE') + history.ntrees = 6 + valid = dict(target=[], ensemble=[], sample_weight=[]) + + for iteration in range(6): + history.after_iteration(dict(num_iter=iteration, data=dict(valid=valid))) + + assert messages == [f'Iter {iteration}; ' for iteration in expected] + + +@pytest.mark.parametrize('verbose', [pytest.param(0, id='zero'), pytest.param(-1, id='negative')]) +@pytest.mark.parametrize('with_eval', [False, True]) +def test_training_without_progress_logging(verbose, with_eval): + rng = np.random.default_rng(42) + X = rng.normal(size=(128, 8)).astype(np.float32) + y = (2 * X[:, 0] - X[:, 1]).astype(np.float32) + eval_sets = [dict(X=X[:64], y=y[:64]), dict(X=X[64:], y=y[64:])] if with_eval else None + params = dict(loss='mse', ntrees=4, max_depth=2, es=0) + model = GradientBoosting(**params, verbose=verbose).fit(X, y, eval_sets=eval_sets) + reference = GradientBoosting(**params, verbose=2).fit(X, y, eval_sets=eval_sets) + + assert len(model.models) == 4 + actual = model.predict(X) + assert np.isfinite(actual).all() + assert np.std(actual) > 0 + np.testing.assert_allclose(actual, reference.predict(X), rtol=1e-5, atol=1e-6) + np.testing.assert_allclose(model.history, reference.history, rtol=1e-5, atol=1e-6) + assert len(model.history) == (4 if with_eval else 0) + if with_eval: + assert all(len(scores) == 2 for scores in model.history) + + +@pytest.mark.parametrize('verbose', [0, -1]) +def test_early_stopping_without_progress_logging(verbose): + X = np.random.default_rng(42).normal(size=(128, 8)).astype(np.float32) + y = np.zeros(len(X), dtype=np.float32) + model = GradientBoosting('mse', ntrees=10, max_depth=2, es=2, verbose=verbose) + model.fit(X, y, eval_sets=[dict(X=X[:64], y=y[:64])]) + + assert len(model.models) == 1 + assert model.best_round == 1 + assert model.history == [[0.0], [0.0], [0.0]] + np.testing.assert_array_equal(model.predict(X), y[:, None])