From 481361701b7e805382036d66f99893cc62160887 Mon Sep 17 00:00:00 2001 From: vsergeev Date: Sat, 24 Oct 2026 19:29:29 +0300 Subject: [PATCH 1/5] Add NaN-aware binary loss and multilabel evaluation metrics --- README.md | 57 +++++++ py_boost/gpu/losses/__init__.py | 8 + py_boost/gpu/losses/losses.py | 35 ++++- py_boost/gpu/losses/metrics.py | 167 +++++++++++++++++++++ tests/test_nan_losses.py | 256 ++++++++++++++++++++++++++++++++ 5 files changed, 522 insertions(+), 1 deletion(-) create mode 100644 tests/test_nan_losses.py diff --git a/README.md b/README.md index d9229c3..c58e73e 100644 --- a/README.md +++ b/README.md @@ -68,3 +68,60 @@ Py-boost is easy to use since it has similar to scikit-learn interface. For usag examples of parsing and inference on CPU with ONNX More examples are coming soon + +## Missing binary targets + +For binary or multilabel targets, use `NaN` for unknown labels and `0` for +observed negatives. `BCEWithNaNLoss` excludes missing labels from gradients, +Hessians and initial class frequencies. An entirely unobserved output starts +at probability 0.5. Keep `lambda_l2 > 0` so leaves without observed labels +have finite updates. + +```python +from py_boost import GradientBoosting +from py_boost.gpu.losses import BCEWithNaNLoss, NaNAucMetric + +model = GradientBoosting(BCEWithNaNLoss(), metric=NaNAucMetric(), lambda_l2=1) +model.fit(X_train, y_train, eval_sets=[{"X": X_valid, "y": y_valid}]) +``` + +The loss also accepts aliases `bce_nan` and `multilabel_nan`. Its default +metric is `NaNBCEMetric`; metric names `bce`/`logloss` and `auc`/`roc` select +the corresponding NaN-aware metrics when used with this loss. Explicit metric +aliases `bce_nan`, `auc_nan` and `gini_nan` are also available. Existing losses +and their default metrics are unchanged. + +Import these metrics from `py_boost.gpu.losses`: + +| Metric | Result | +| --- | --- | +| `NaNBCEMetric` | Mean BCE over observed labels; `return_list=True` gives per-output losses as a CuPy array. | +| `BCEWithNaNMetricWeighted(output_weights)` | Weighted mean of per-output BCE. `BCEwithNaNMetricWeighted` is an alias. | +| `NaNAucMetric` | Macro ROC AUC; `return_list=True` gives a NumPy array. | +| `NaNAucMetricByClass` | Per-output ROC AUC as a NumPy array, for reporting. | +| `NaNGiniMetricByClass` | Per-output Gini (`2 * AUC - 1`) as a NumPy array, for reporting. | +| `NaNGiniMetricAtN` | Weighted macro Gini; call with `prod_weights=` and `n=` to restrict to the first `n` target columns. | + +All metrics support nonnegative row weights of shape `(n_samples,)` or +`(n_samples, 1)`. NaN labels and zero-weight rows are excluded independently +for each target. ROC AUC/Gini are undefined for outputs without positive +weight in both classes; BCE is undefined for outputs without any positively +weighted labels. Per-output results contain NaN for these outputs, and macro +metrics exclude them. A scalar metric is NaN if no eligible output remains. +Validate that your evaluation set has enough observed labels for early stopping; +the per-class vector metrics are for reporting only. + +Observed targets must be binary. Predictions must be finite wherever a label +has positive row weight; invalid predictions are reported as errors rather +than treated as missing labels. BCE additionally requires probabilities in +`[0, 1]`, while ROC AUC/Gini also accept finite decision scores. + +Output/product weights must follow target column order and are normalized +over eligible, positively weighted outputs. `n` selects leading columns, +not the outputs with the highest scores. Compared with custom implementations +that return an unnormalized weighted sum, non-unit-sum weights therefore have +different semantics. Standard `CrossValidation` is supported; `AdaptiveESCV` +requires an explicit scorer for missing labels. + +GPU regression tests can be run with `python -m pytest -q tests/test_nan_losses.py` +after installing Py-Boost, a compatible CuPy build, and pytest. diff --git a/py_boost/gpu/losses/__init__.py b/py_boost/gpu/losses/__init__.py index 10f78ed..4b4b851 100644 --- a/py_boost/gpu/losses/__init__.py +++ b/py_boost/gpu/losses/__init__.py @@ -9,6 +9,7 @@ 'MSELoss', 'MSLELoss', 'BCELoss', + 'BCEWithNaNLoss', 'CrossEntropyLoss', 'Metric', @@ -16,6 +17,13 @@ 'RMSLEMetric', 'R2Score', 'BCEMetric', + 'NaNBCEMetric', + 'BCEWithNaNMetricWeighted', + 'BCEwithNaNMetricWeighted', + 'NaNAucMetric', + 'NaNAucMetricByClass', + 'NaNGiniMetricByClass', + 'NaNGiniMetricAtN', 'AccuracyMetric', 'RocAucMetric', diff --git a/py_boost/gpu/losses/losses.py b/py_boost/gpu/losses/losses.py index 4287815..d5e82f5 100644 --- a/py_boost/gpu/losses/losses.py +++ b/py_boost/gpu/losses/losses.py @@ -7,7 +7,7 @@ except Exception: CUDA_FOUND = False -from .metrics import metric_alias, RMSEMetric, RMSLEMetric, BCEMetric +from .metrics import metric_alias, RMSEMetric, RMSLEMetric, BCEMetric, NaNBCEMetric, NaNAucMetric from .multiclass_metrics import multiclass_metric_alias, CrossEntropyMetric @@ -154,6 +154,36 @@ def get_metric_from_string(self, name=None): return metric_alias[name] +class BCEWithNaNLoss(BCELoss): + """Binary/multilabel log loss with missing (NaN) targets. + + Missing labels contribute zero gradient and Hessian. Initial log odds + use only observed labels; an entirely missing output starts at zero + (probability 0.5). Use positive lambda_l2 for leaves with no observed + labels. The default evaluation metric also ignores missing labels. + """ + + def base_score(self, y_true): + observed = ~cp.isnan(y_true) + counts = observed.sum(axis=0) + means = cp.where(observed, y_true, 0).sum(axis=0) / cp.maximum(counts, 1).astype(y_true.dtype) + means = cp.where(counts > 0, means, 0.5) + means = cp.clip(means, self.clip_value, 1 - self.clip_value) + return cp.log(means / (1 - means)) + + def get_grad_hess(self, y_true, y_pred): + missing = cp.isnan(y_true) + grad, hess = super().get_grad_hess(cp.where(missing, 0, y_true), y_pred) + return cp.where(missing, 0, grad), cp.where(missing, 0, hess) + + def get_metric_from_string(self, name=None): + if name is None or name in ('bce', 'logloss'): + return NaNBCEMetric() + if name in ('auc', 'roc'): + return NaNAucMetric() + return metric_alias[name] + + def softmax(x, clip_val=1e-5): xp = np if type(x) is np.ndarray else cp @@ -225,6 +255,9 @@ def get_metric_from_string(self, name=None): 'multilabel': BCELoss(), 'logloss': BCELoss(), + 'bce_nan': BCEWithNaNLoss(), + 'multilabel_nan': BCEWithNaNLoss(), + # for multiclass 'multiclass': CrossEntropyLoss(), 'crossentropy': CrossEntropyLoss(), diff --git a/py_boost/gpu/losses/metrics.py b/py_boost/gpu/losses/metrics.py index a2f4c44..91c4b1e 100644 --- a/py_boost/gpu/losses/metrics.py +++ b/py_boost/gpu/losses/metrics.py @@ -1,5 +1,7 @@ """Common metrics""" +import numpy as np + try: import cupy as cp except Exception: @@ -201,6 +203,167 @@ def compare(self, v0, v1): return v0 > v1 +def _nan_metric_inputs(y_true, y_pred, sample_weight): + """Validate shapes and prepare row weights for metrics with missing labels.""" + y_true, y_pred = cp.asarray(y_true), cp.asarray(y_pred) + if y_true.ndim == 1: + y_true = y_true[:, None] + if y_pred.ndim == 1: + y_pred = y_pred[:, None] + if y_true.ndim != 2 or y_true.shape != y_pred.shape or y_true.shape[1] == 0: + raise ValueError('Targets and predictions must have the same nonempty output dimension') + + if sample_weight is None: + weights = cp.ones((y_true.shape[0], 1), dtype=cp.float64) + else: + weights = cp.asarray(sample_weight, dtype=cp.float64) + if weights.ndim == 1: + weights = weights[:, None] + if weights.shape != (y_true.shape[0], 1): + raise ValueError('sample_weight must have shape (n_samples,) or (n_samples, 1)') + if not bool(cp.isfinite(weights).all()) or bool((weights < 0).any()): + raise ValueError('sample_weight must be finite and nonnegative') + observed = ~cp.isnan(y_true) & (weights > 0) + if bool((observed & (y_true != 0) & (y_true != 1)).any()): + raise ValueError('Observed binary targets must be 0 or 1') + if bool((observed & ~cp.isfinite(y_pred)).any()): + raise ValueError('Predictions must be finite at observed, positive-weight labels') + return y_true, y_pred, weights + + +def _output_weights(weights): + weights = np.array(weights, dtype=np.float64, copy=True) + if (weights.ndim != 1 or not np.isfinite(weights).all() + or (weights < 0).any() or not (weights > 0).any()): + raise ValueError('Output weights must be a finite, nonnegative vector with positive total weight') + return weights + + +def _nan_weighted_mean(values, weights=None): + """Average defined outputs, preserving infinite errors for incorrect predictions.""" + if weights is None: + weights = np.ones(values.shape[0], dtype=np.float64) + if weights.shape != values.shape: + raise ValueError('There must be one output weight per target column') + valid = ~np.isnan(values) & (weights > 0) + if not valid.any(): + return float('nan') + # Rescale first to avoid overflow for large finite output weights. + weights = weights[valid] / weights[valid].max() + return float(np.sum(values[valid] * weights) / weights.sum()) + + +class NaNBCEMetric(BCEMetric): + """Binary log loss averaged over observed labels, ignoring NaN targets. + + Row weights may have shape (n_samples,) or (n_samples, 1). With + return_list=True, return a CuPy vector of per-output losses. An output + with no positively weighted labels has NaN loss; the scalar result is + NaN only when there are no observed labels with positive weight. + """ + alias = 'BCE_nan' + + def __call__(self, y_true, y_pred, sample_weight=None, return_list=False): + y_true, y_pred, weights = _nan_metric_inputs(y_true, y_pred, sample_weight) + observed = ~cp.isnan(y_true) & (weights > 0) + if bool((observed & ((y_pred < 0) | (y_pred > 1))).any()): + raise ValueError('BCE predictions must be probabilities between 0 and 1') + # Mask before evaluating the loss: NaN/inf at missing labels must not + # contaminate the result, and multiplying an infinite error by zero + # would still produce NaN. + err = self.error(cp.where(observed, y_true, 0), cp.where(observed, y_pred, 0.5)) + numerator = (cp.where(observed, err, 0) * weights).sum(axis=0) + denominator = (observed * weights).sum(axis=0) + if return_list: + return numerator / cp.where(denominator > 0, denominator, cp.nan) + total = denominator.sum() + return float(numerator.sum() / cp.where(total > 0, total, cp.nan)) + + +class BCEWithNaNMetricWeighted(NaNBCEMetric): + """Weighted mean of per-output NaN-aware binary log losses. + + Output weights are normalized over outputs with observed labels and + positive weight. If no such output remains, return NaN. + """ + + def __init__(self, output_weights): + self.output_weights = _output_weights(output_weights) + + def __call__(self, y_true, y_pred, sample_weight=None): + errors = super().__call__(y_true, y_pred, sample_weight, return_list=True).get() + return _nan_weighted_mean(errors, self.output_weights) + + +# Retain the spelling used by existing custom-metric implementations. +BCEwithNaNMetricWeighted = BCEWithNaNMetricWeighted + + +class NaNAucMetric(Metric): + """Macro ROC AUC over binary outputs with missing targets. + + NaN labels and zero-weight rows are excluded independently per output. + Outputs without positive weight in both classes have undefined AUC + (NaN) and are excluded from the macro average. Return NaN when every + output is undefined. return_list=True returns a NumPy vector, including + NaNs for undefined outputs. Sample weights are passed to each ROC AUC. + """ + alias = 'AUC_nan' + + def __call__(self, y_true, y_pred, sample_weight=None, return_list=False): + y_true, y_pred, weights = _nan_metric_inputs(y_true, y_pred, sample_weight) + scores = np.full(y_true.shape[1], np.nan, dtype=np.float64) + for i in range(y_true.shape[1]): + observed = ~cp.isnan(y_true[:, i]) & (weights[:, 0] > 0) + labels = y_true[observed, i] + w = weights[observed, 0] + if not bool((labels == 0).any()) or not bool((labels == 1).any()): + continue + scores[i] = auc(labels, y_pred[observed, i], w) + return scores if return_list else _nan_weighted_mean(scores) + + def compare(self, v0, v1): + return v0 > v1 + + +class NaNAucMetricByClass(NaNAucMetric): + """Per-output ROC AUC as a NumPy vector; for reporting, not early stopping.""" + + def __call__(self, y_true, y_pred, sample_weight=None): + return super().__call__(y_true, y_pred, sample_weight, return_list=True) + + +class NaNGiniMetricByClass(NaNAucMetricByClass): + """Per-output Gini (2 * AUC - 1); for reporting, not early stopping.""" + alias = 'Gini_nan' + + def __call__(self, y_true, y_pred, sample_weight=None): + return 2 * super().__call__(y_true, y_pred, sample_weight) - 1 + + +class NaNGiniMetricAtN(NaNAucMetric): + """Weighted macro Gini over the first n outputs (all outputs by default). + + prod_weights has one finite, nonnegative weight per target column and + is normalized over selected outputs with defined AUC and positive + weight. Return NaN if none remain. n selects by column order, not score. + """ + alias = 'Gini_nan' + + def __call__(self, y_true, y_pred, sample_weight=None, prod_weights=None, n=None): + scores = super().__call__(y_true, y_pred, sample_weight, return_list=True) + if n is None: + n = len(scores) + if isinstance(n, (bool, np.bool_)) or not isinstance(n, (int, np.integer)) or not 1 <= n <= len(scores): + raise ValueError('n must be an integer between 1 and the number of outputs') + weights = None if prod_weights is None else _output_weights(prod_weights) + if weights is not None: + if weights.shape != scores.shape: + raise ValueError('There must be one output weight per target column') + weights = weights[:n] + return _nan_weighted_mean(2 * scores[:n] - 1, weights) + + class ThresholdMetrics(Metric): """Basic class to handle metrics, that accept class label prediction as input""" @@ -320,6 +483,10 @@ def __call__(self, y_true, y_pred, sample_weight=None): 'bce': BCEMetric(), 'logloss': BCEMetric(), + 'bce_nan': NaNBCEMetric(), + 'auc_nan': NaNAucMetric(), + 'gini_nan': NaNGiniMetricAtN(), + 'precision': Precision(), 'recall': Recall(), 'f1_score': F1Score(), diff --git a/tests/test_nan_losses.py b/tests/test_nan_losses.py new file mode 100644 index 0000000..c5d2022 --- /dev/null +++ b/tests/test_nan_losses.py @@ -0,0 +1,256 @@ +"""Numerical and training checks for binary losses with missing labels.""" + +import numpy as np +import pytest +from sklearn.metrics import log_loss, roc_auc_score + + +@pytest.fixture(scope='module') +def losses(): + cp = pytest.importorskip('cupy') + try: + count = cp.cuda.runtime.getDeviceCount() + except cp.cuda.runtime.CUDARuntimeError as exc: + pytest.skip(f'CUDA device unavailable: {exc}') + if not count: + pytest.skip('A CUDA device is required') + from py_boost.gpu import losses + return losses + + +@pytest.fixture +def data(): + y = np.array([ + [0, 1, np.nan], [1, np.nan, 0], [0, 0, 1], + [1, 1, 0], [np.nan, 0, 1], [1, 0, np.nan], + ], dtype=np.float64) + p = np.array([ + [.1, .8, np.nan], [.3, np.nan, .9], [.3, .7, .8], + [.8, .5, .4], [np.nan, .1, .3], [.7, .5, np.nan], + ]) + return y, p, np.array([1, 5, 2, 0, 3, 4], dtype=np.float64) + + +def test_gradients_and_hessians_match_finite_differences(losses, data): + cp = losses.cp + y, _, _ = data + logits = np.linspace(-2, 2, y.size).reshape(y.shape) + loss = losses.BCEWithNaNLoss() + grad, hess = loss.get_grad_hess(cp.asarray(y), cp.asarray(logits)) + + def objective(raw): + return np.where(np.isnan(y), 0, np.logaddexp(0, raw) - np.nan_to_num(y) * raw) + + step = 1e-4 + plus, center, minus = objective(logits + step), objective(logits), objective(logits - step) + np.testing.assert_allclose(grad.get(), (plus - minus) / (2 * step), atol=1e-8) + np.testing.assert_allclose(hess.get(), (plus - 2 * center + minus) / step ** 2, atol=1e-7) + + +@pytest.mark.parametrize('constant', [None, 0, 1]) +def test_without_missing_labels_matches_bce(losses, constant): + cp = losses.cp + y = cp.asarray([[0, 1], [1, 0], [1, 1]], dtype=cp.float32) + if constant is not None: + y[:] = constant + raw = cp.asarray([[-2, 1], [0, 2], [3, -1]], dtype=cp.float32) + old, new = losses.BCELoss(), losses.BCEWithNaNLoss() + np.testing.assert_allclose(old.base_score(y).get(), new.base_score(y).get(), rtol=1e-6) + for left, right in zip(old(y, raw), new(y, raw)): + np.testing.assert_array_equal(left.get(), right.get()) + prediction = new.postprocess_output(raw) + np.testing.assert_allclose( + float(losses.NaNBCEMetric()(y, prediction)), float(losses.BCEMetric()(y, prediction)), rtol=1e-6, + ) + + +def test_missing_output_has_neutral_prior_and_zero_derivatives(losses): + cp = losses.cp + y = cp.asarray([[1, cp.nan, 0], [0, cp.nan, 0], [cp.nan, cp.nan, 1]], dtype=cp.float32) + loss = losses.BCEWithNaNLoss() + expected = np.array([0, 0, np.log(0.5)], dtype=np.float32) + np.testing.assert_allclose(loss.base_score(y).get(), expected, atol=1e-6) + raw = cp.where(cp.isnan(y), cp.nan, 0).astype(cp.float32) + grad, hess = loss(y, raw) + assert bool(cp.isfinite(grad).all()) and bool(cp.isfinite(hess).all()) + assert bool((grad[cp.isnan(y)] == 0).all()) + assert bool((hess[cp.isnan(y)] == 0).all()) + + +@pytest.mark.parametrize('weighted', [False, True]) +@pytest.mark.parametrize('column_weights', [False, True]) +def test_bce_matches_observed_label_reference(losses, data, weighted, column_weights): + y, p, w = data + weights = w if weighted else np.ones(len(y)) + passed_weights = None if not weighted else weights[:, None] if column_weights else weights + observed = ~np.isnan(y) + rows, cols = np.nonzero(observed) + expected = log_loss(y[rows, cols], p[rows, cols], sample_weight=weights[rows], labels=[0, 1]) + metric = losses.NaNBCEMetric() + assert metric(y, p, passed_weights) == pytest.approx(expected) + per_output = metric(y, p, passed_weights, return_list=True).get() + expected_outputs = [ + log_loss(y[observed[:, i], i], p[observed[:, i], i], + sample_weight=weights[observed[:, i]], labels=[0, 1]) + for i in range(y.shape[1]) + ] + np.testing.assert_allclose(per_output, expected_outputs) + output_weights = np.array([1, 3, 2]) + weighted_metric = losses.BCEWithNaNMetricWeighted(output_weights) + assert weighted_metric(y, p, passed_weights) == pytest.approx(np.average(expected_outputs, weights=output_weights)) + assert weighted_metric.compare(0.1, 0.2) + + +@pytest.mark.parametrize('weighted', [False, True]) +@pytest.mark.parametrize('column_weights', [False, True]) +def test_auc_and_gini_match_sklearn(losses, data, weighted, column_weights): + y, p, w = data + weights = w if weighted else np.ones(len(y)) + passed_weights = None if not weighted else weights[:, None] if column_weights else weights + expected = [] + for i in range(y.shape[1]): + observed = ~np.isnan(y[:, i]) & (weights > 0) + expected.append(roc_auc_score(y[observed, i], p[observed, i], sample_weight=weights[observed])) + expected = np.array(expected) + metric = losses.NaNAucMetric() + assert metric(y, p, passed_weights) == pytest.approx(expected.mean()) + np.testing.assert_allclose(metric(y, p, passed_weights, return_list=True), expected) + np.testing.assert_allclose(losses.NaNAucMetricByClass()(y, p, passed_weights), expected) + np.testing.assert_allclose(losses.NaNGiniMetricByClass()(y, p, passed_weights), 2 * expected - 1) + gini = losses.NaNGiniMetricAtN() + assert gini(y, p, passed_weights) == pytest.approx(2 * expected.mean() - 1) + assert gini(y, p, passed_weights, prod_weights=[1, 3, 2], n=2) == pytest.approx( + np.average(2 * expected[:2] - 1, weights=[1, 3]), + ) + assert metric.compare(0.8, 0.7) and gini.compare(0.8, 0.7) + + +def test_undefined_outputs_and_zero_weight_rows(losses): + y = np.array([[np.nan, 1, 0, 0], [np.nan, 1, 1, 1], [np.nan, 1, 1, 0]]) + p = np.array([[np.nan, .5, .5, .1], [np.inf, .5, .5, .8], [-np.inf, .5, .5, np.nan]]) + weights = np.array([1, 1, 0]) + expected_auc = [np.nan, np.nan, 0.5, 1] + np.testing.assert_allclose(losses.NaNAucMetricByClass()(y, p, weights), expected_auc, equal_nan=True) + assert losses.NaNAucMetric()(y, p, weights) == pytest.approx(0.75) + assert losses.NaNGiniMetricAtN()(y, p, weights, prod_weights=[100, 100, 1, 3]) == pytest.approx(0.75) + assert np.isnan(losses.NaNGiniMetricAtN()(y, p, weights, n=2)) + errors = losses.NaNBCEMetric()(y, p, weights, return_list=True).get() + assert np.isnan(errors[0]) and np.isfinite(errors[1:]).all() + assert losses.BCEWithNaNMetricWeighted([100, 0, 1, 3])(y, p, weights) == pytest.approx( + np.average(errors[2:], weights=[1, 3]), + ) + assert np.isnan(losses.NaNBCEMetric()(y, p, np.zeros(3))) + assert np.isnan(losses.NaNAucMetric()(y, p, np.zeros(3))) + assert np.isnan(losses.BCEWithNaNMetricWeighted([1, 0, 0, 0])(y, p, weights)) + assert np.isinf(losses.NaNBCEMetric()([1], [0])) + + +@pytest.mark.parametrize('metric', ['NaNBCEMetric', 'NaNAucMetric', 'NaNGiniMetricAtN']) +def test_empty_and_single_output_inputs(losses, metric): + metric = getattr(losses, metric)() + assert np.isnan(metric(np.empty((0, 2)), np.empty((0, 2)))) + assert np.isnan(metric([np.nan], [np.nan])) + assert np.isfinite(metric([0, 1], [.1, .9])) + + +@pytest.mark.parametrize('metric', ['NaNBCEMetric', 'NaNAucMetric']) +@pytest.mark.parametrize('prediction', [np.nan, np.inf, -np.inf]) +def test_invalid_observed_predictions_are_not_hidden(losses, metric, prediction): + with pytest.raises(ValueError, match='Predictions must be finite'): + getattr(losses, metric)()([[0, 1], [1, 0]], [[prediction, .8], [.7, .2]]) + + +@pytest.mark.parametrize('label', [-1, 2, np.inf]) +def test_invalid_binary_targets(losses, label): + with pytest.raises(ValueError, match='binary targets'): + losses.NaNAucMetric()([0, label], [.1, .8]) + + +@pytest.mark.parametrize('prediction', [-.1, 1.1]) +def test_bce_requires_probabilities(losses, prediction): + with pytest.raises(ValueError, match='probabilities'): + losses.NaNBCEMetric()([0, 1], [.1, prediction]) + + +@pytest.mark.parametrize('weights', [[1, -1], [0, 0], [1, np.nan], [1, np.inf], [[1, 2]]]) +def test_invalid_output_weights(losses, weights): + with pytest.raises(ValueError, match='weights'): + losses.BCEWithNaNMetricWeighted(weights) + with pytest.raises(ValueError, match='weights'): + losses.NaNGiniMetricAtN()([[0, 1], [1, 0]], [[.1, .9], [.9, .1]], prod_weights=weights) + + +@pytest.mark.parametrize('metric', ['NaNBCEMetric', 'NaNAucMetric']) +@pytest.mark.parametrize('weights', [[1, -1], [1, np.nan], [1, np.inf], [1], [[1, 1], [1, 1]]]) +def test_invalid_sample_weights(losses, metric, weights): + with pytest.raises(ValueError, match='sample_weight'): + getattr(losses, metric)()([[0], [1]], [[.1], [.9]], weights) + + +@pytest.mark.parametrize('n', [0, -1, 3, 1.5, True]) +def test_invalid_output_count(losses, n): + with pytest.raises(ValueError, match='n must'): + losses.NaNGiniMetricAtN()([[0, 1], [1, 0]], [[.1, .9], [.9, .1]], n=n) + + +def test_shapes_and_aliases(losses): + with pytest.raises(ValueError, match='same nonempty'): + losses.NaNBCEMetric()(np.zeros((2, 2)), np.zeros((2, 1))) + with pytest.raises(ValueError, match='one output weight'): + losses.BCEWithNaNMetricWeighted([1, 2])([0, 1], [.1, .9]) + with pytest.raises(ValueError, match='one output weight'): + losses.NaNGiniMetricAtN()([0, 1], [.1, .9], prod_weights=[1, 2]) + assert losses.BCEwithNaNMetricWeighted is losses.BCEWithNaNMetricWeighted + assert isinstance(losses.loss_alias['bce_nan'], losses.BCEWithNaNLoss) + assert isinstance(losses.loss_alias['multilabel_nan'], losses.BCEWithNaNLoss) + assert type(losses.loss_alias['bce']) is losses.BCELoss + assert type(losses.metric_alias['bce']) is losses.BCEMetric + assert isinstance(losses.metric_alias['gini_nan'], losses.NaNGiniMetricAtN) + for name in [None, 'bce', 'logloss', 'bce_nan']: + assert isinstance(losses.BCEWithNaNLoss().get_metric_from_string(name), losses.NaNBCEMetric) + for name in ['auc', 'roc', 'auc_nan']: + assert isinstance(losses.BCEWithNaNLoss().get_metric_from_string(name), losses.NaNAucMetric) + for name in ['BCEWithNaNLoss', 'NaNBCEMetric', 'NaNAucMetric', 'NaNAucMetricByClass', + 'NaNGiniMetricByClass', 'NaNGiniMetricAtN', 'BCEWithNaNMetricWeighted', 'BCEwithNaNMetricWeighted']: + assert name in losses.__all__ + + +@pytest.mark.parametrize('sketch', [None, 1, 2]) +@pytest.mark.parametrize('metric', [None, 'auc', 'gini_nan']) +def test_training_missing_targets(losses, sketch, metric): + from py_boost import GradientBoosting + from py_boost.multioutput.sketching import RandomProjectionSketch + + rng = np.random.default_rng(12) + X = rng.normal(size=(128, 6)).astype(np.float32) + y = (X[:, :3] > 0).astype(np.float32) + y[rng.random(y.shape) < 0.25] = np.nan + y[:, 2] = np.nan # No labels in an entire output must not poison the ensemble. + model = GradientBoosting( + 'bce_nan', metric=metric, ntrees=5, max_depth=2, min_data_in_leaf=2, + max_bin=16, es=2, verbose=100, + multioutput_sketch=None if sketch is None else RandomProjectionSketch(sketch), + ) + model.fit(X[:96], y[:96], eval_sets=[{'X': X[96:], 'y': y[96:]}]) + prediction = model.predict(X[96:]) + assert prediction.shape == y[96:].shape + assert np.isfinite(prediction).all() and np.isfinite(model.history).all() + assert ((prediction > 0) & (prediction < 1)).all() + np.testing.assert_array_equal(prediction[:, 2], np.full(32, 0.5)) + assert np.std(prediction[:, 0]) > 0 + + +def test_cross_validation_missing_targets(losses): + from py_boost import GradientBoosting + from py_boost.cv import CrossValidation + + rng = np.random.default_rng(12) + X = rng.normal(size=(128, 4)).astype(np.float32) + y = (X[:, :2] > 0).astype(np.float32) + y[rng.random(y.shape) < .25] = np.nan + model = GradientBoosting('bce_nan', ntrees=3, max_depth=2, max_bin=16, verbose=100) + cv = CrossValidation(model) + predictions = cv.fit_predict(X, y, cv=np.arange(len(y)) % 2) + assert predictions.shape == y.shape and np.isfinite(predictions).all() + assert np.isfinite(cv.predict(X)).all() + assert len(cv.models) == 2 From af3243ea1b4c7a37b6add256ef8dab3346ad8b4f Mon Sep 17 00:00:00 2001 From: vsergeev Date: Wed, 28 Oct 2026 18:22:02 +0300 Subject: [PATCH 2/5] Address NaN-target loss review and stabilize weighted evaluation --- README.md | 37 ++- py_boost/callbacks/callback.py | 8 + py_boost/gpu/losses/__init__.py | 1 - py_boost/gpu/losses/losses.py | 19 +- py_boost/gpu/losses/metrics.py | 48 ++-- py_boost/multioutput/sketching.py | 49 +++- tests/test_nan_losses.py | 256 +++++++++++++++++- .../Tutorial_4_Handle_null_targets.ipynb | 191 +++++-------- 8 files changed, 429 insertions(+), 180 deletions(-) diff --git a/README.md b/README.md index 61c8fb2..12d08c2 100644 --- a/README.md +++ b/README.md @@ -73,9 +73,11 @@ More examples are coming soon. For binary or multilabel targets, use `NaN` for unknown labels and `0` for observed negatives. `BCEWithNaNLoss` excludes missing labels from gradients, -Hessians and initial class frequencies. An entirely unobserved output starts -at probability 0.5. Keep `lambda_l2 > 0` so leaves without observed labels -have finite updates. +Hessians and initial class frequencies. Every training target column must +contain at least one observed label. Otherwise, training raises `ValueError` +listing the zero-based indices of the affected columns; no columns are +silently removed. Entirely unobserved columns are allowed in evaluation sets. +Keep `lambda_l2 > 0` so leaves without observed labels have finite updates. ```python from py_boost import GradientBoosting @@ -96,22 +98,31 @@ Import these metrics from `py_boost.gpu.losses`: | Metric | Result | | --- | --- | | `NaNBCEMetric` | Mean BCE over observed labels; `return_list=True` gives per-output losses as a CuPy array. | -| `BCEWithNaNMetricWeighted(output_weights)` | Weighted mean of per-output BCE. `BCEwithNaNMetricWeighted` is an alias. | +| `BCEWithNaNMetricWeighted(output_weights)` | Weighted mean of per-output BCE. | | `NaNAucMetric` | Macro ROC AUC; `return_list=True` gives a NumPy array. | | `NaNAucMetricByClass` | Per-output ROC AUC as a NumPy array, for reporting. | | `NaNGiniMetricByClass` | Per-output Gini (`2 * AUC - 1`) as a NumPy array, for reporting. | | `NaNGiniMetricAtN` | Weighted macro Gini; call with `prod_weights=` and `n=` to restrict to the first `n` target columns. | -All metrics support nonnegative row weights of shape `(n_samples,)` or +All metrics support finite, nonnegative row weights of shape `(n_samples,)` or `(n_samples, 1)`. NaN labels and zero-weight rows are excluded independently for each target. ROC AUC/Gini are undefined for outputs without positive weight in both classes; BCE is undefined for outputs without any positively weighted labels. Per-output results contain NaN for these outputs, and macro metrics exclude them. A scalar metric is NaN if no eligible output remains. -Validate that your evaluation set has enough observed labels for early stopping; -the per-class vector metrics are for reporting only. - -Observed targets must be binary. Predictions must be finite wherever a label +Positive row weights are rescaled before aggregation so multiplying all +weights by a common positive constant does not change the metric, subject +to floating-point precision. All-zero weights produce an undefined metric. + +With early stopping enabled, an undefined (NaN) scalar metric on the last +evaluation set raises a clear error instead of silently pruning to one tree. +Choose an evaluation set and metric with enough observed, positively weighted +labels, or set `es=0` to keep training and record NaN in the evaluation history. +The per-class vector metrics are for reporting only. + +Observed targets must be `0` or `1`; `NaN` is the only missing-label marker. +Training and evaluation labels are validated once before training, including +labels on zero-weight rows. Predictions must be finite wherever a label has positive row weight; invalid predictions are reported as errors rather than treated as missing labels. BCE additionally requires probabilities in `[0, 1]`, while ROC AUC/Gini also accept finite decision scores. @@ -123,5 +134,13 @@ that return an unnormalized weighted sum, non-unit-sum weights therefore have different semantics. Standard `CrossValidation` is supported; `AdaptiveESCV` requires an explicit scorer for missing labels. +`FilterSketch` supports zero Hessians from missing or zero-weight labels. +Outputs with zero total Hessian are excluded from selection. If all remaining +output importances are zero, selection uses a uniform distribution over the +eligible outputs; if none remain, it raises an error identifying the lack of +positive Hessian mass. The training-column check counts observed labels before +row weights, so zero-weight observations alone do not make an output eligible +for `FilterSketch` with `use_hess=True`. + GPU regression tests can be run with `python -m pytest -q tests/test_nan_losses.py` after installing Py-Boost, a compatible CuPy build, and pytest. diff --git a/py_boost/callbacks/callback.py b/py_boost/callbacks/callback.py index 7817b69..66c7c98 100644 --- a/py_boost/callbacks/callback.py +++ b/py_boost/callbacks/callback.py @@ -1,5 +1,6 @@ """Default callbacks""" import logging +import math from ..utils.logging import verbosity_to_loglevel, set_stdout_level logger = logging.getLogger(__name__) @@ -253,6 +254,13 @@ def after_iteration(self, build_info): num_iter = build_info['num_iter'] # if multiple valid sets passed - use the last one score = build_info['iter_score'][-1] + if math.isnan(score): + raise ValueError( + f"Early stopping metric '{self.metric.alias}' is undefined (NaN) on " + f"evaluation set {len(build_info['iter_score']) - 1}. Check observed labels " + 'and positive sample weights, choose another metric or evaluation set, ' + 'or disable early stopping with es=0.' + ) if num_iter == 0: self.best_score = score diff --git a/py_boost/gpu/losses/__init__.py b/py_boost/gpu/losses/__init__.py index 4b4b851..3c5a280 100644 --- a/py_boost/gpu/losses/__init__.py +++ b/py_boost/gpu/losses/__init__.py @@ -19,7 +19,6 @@ 'BCEMetric', 'NaNBCEMetric', 'BCEWithNaNMetricWeighted', - 'BCEwithNaNMetricWeighted', 'NaNAucMetric', 'NaNAucMetricByClass', 'NaNGiniMetricByClass', diff --git a/py_boost/gpu/losses/losses.py b/py_boost/gpu/losses/losses.py index d5e82f5..d9b02d9 100644 --- a/py_boost/gpu/losses/losses.py +++ b/py_boost/gpu/losses/losses.py @@ -158,16 +158,25 @@ class BCEWithNaNLoss(BCELoss): """Binary/multilabel log loss with missing (NaN) targets. Missing labels contribute zero gradient and Hessian. Initial log odds - use only observed labels; an entirely missing output starts at zero - (probability 0.5). Use positive lambda_l2 for leaves with no observed - labels. The default evaluation metric also ignores missing labels. + use only observed labels. Every training output must have at least one + observed label; entirely missing evaluation outputs are allowed. Use + positive lambda_l2 for leaves with no observed labels. The default + evaluation metric also ignores missing labels. """ + def preprocess_input(self, y_true): + if bool((~cp.isnan(y_true) & (y_true != 0) & (y_true != 1)).any()): + raise ValueError('Observed binary targets must be 0 or 1; use NaN for missing labels') + return y_true + def base_score(self, y_true): observed = ~cp.isnan(y_true) counts = observed.sum(axis=0) - means = cp.where(observed, y_true, 0).sum(axis=0) / cp.maximum(counts, 1).astype(y_true.dtype) - means = cp.where(counts > 0, means, 0.5) + missing_outputs = cp.flatnonzero(counts == 0).get().tolist() + if missing_outputs: + raise ValueError('Training target columns have no observed labels ' + f'(zero-based indices): {missing_outputs}') + means = cp.where(observed, y_true, 0).sum(axis=0) / counts.astype(y_true.dtype) means = cp.clip(means, self.clip_value, 1 - self.clip_value) return cp.log(means / (1 - means)) diff --git a/py_boost/gpu/losses/metrics.py b/py_boost/gpu/losses/metrics.py index 91c4b1e..fd9ec5e 100644 --- a/py_boost/gpu/losses/metrics.py +++ b/py_boost/gpu/losses/metrics.py @@ -159,22 +159,27 @@ def auc(y, x, sample_weight=None): return 0.5 if sample_weight is None: - sample_weight = cp.ones_like(y) + sample_weight = cp.ones(y.shape, dtype=cp.float64) + + # AUC is invariant to separate rescaling of the two classes. Normalize + # before summation to avoid overflow/underflow, even for imbalanced weights. + positive_weight = cp.asarray(sample_weight, dtype=cp.float64) * y + negative_weight = cp.asarray(sample_weight, dtype=cp.float64) * (1 - y) + positive_scale, negative_scale = positive_weight.max(), negative_weight.max() + positive_weight /= cp.where(positive_scale > 0, positive_scale, 1) + negative_weight /= cp.where(negative_scale > 0, negative_scale, 1) rank_x = cp.searchsorted(unique_x, x) sum_1 = cp.zeros_like(unique_x, dtype=cp.float64) - sum_1.scatter_add(rank_x, sample_weight * y) + sum_1.scatter_add(rank_x, positive_weight) sum_0 = cp.zeros_like(unique_x, dtype=cp.float64) - sum_0.scatter_add(rank_x, sample_weight * (1 - y)) - - cs_0 = sum_0.cumsum() - auc_ = (cs_0 - sum_0 / 2) * sum_1 - - tot = cs_0[-1] * sum_1.sum() + sum_0.scatter_add(rank_x, negative_weight) - return float(auc_.sum() / tot) + sum_0 /= sum_0.sum() + sum_1 /= sum_1.sum() + return float(((sum_0.cumsum() - sum_0 / 2) * sum_1).sum()) class RocAucMetric(Metric): @@ -272,12 +277,21 @@ def __call__(self, y_true, y_pred, sample_weight=None, return_list=False): # contaminate the result, and multiplying an infinite error by zero # would still produce NaN. err = self.error(cp.where(observed, y_true, 0), cp.where(observed, y_pred, 0.5)) - numerator = (cp.where(observed, err, 0) * weights).sum(axis=0) - denominator = (observed * weights).sum(axis=0) - if return_list: - return numerator / cp.where(denominator > 0, denominator, cp.nan) - total = denominator.sum() - return float(numerator.sum() / cp.where(total > 0, total, cp.nan)) + # Exclude missing rows before selecting the scale. Per-output metrics + # need independent scales so large weights in other outputs cannot + # underflow all weights in an otherwise well-defined output. + weights = cp.where(observed, weights, 0) + axis = 0 if return_list else None + if weights.shape[0]: + scale = weights.max(axis=axis) + weights /= cp.where(scale > 0, scale, 1) + # An impossible prediction still has infinite loss at any positive + # original weight, even if rescaling underflows that weight to zero. + weighted_error = cp.where(observed & cp.isinf(err), cp.inf, err * weights) + numerator = weighted_error.sum(axis=axis) + denominator = weights.sum(axis=axis) + result = numerator / cp.where(denominator > 0, denominator, cp.nan) + return result if return_list else float(result) class BCEWithNaNMetricWeighted(NaNBCEMetric): @@ -295,10 +309,6 @@ def __call__(self, y_true, y_pred, sample_weight=None): return _nan_weighted_mean(errors, self.output_weights) -# Retain the spelling used by existing custom-metric implementations. -BCEwithNaNMetricWeighted = BCEWithNaNMetricWeighted - - class NaNAucMetric(Metric): """Macro ROC AUC over binary outputs with missing targets. diff --git a/py_boost/multioutput/sketching.py b/py_boost/multioutput/sketching.py index e877b97..83bd4da 100644 --- a/py_boost/multioutput/sketching.py +++ b/py_boost/multioutput/sketching.py @@ -164,7 +164,12 @@ def __call__(self, grad, hess): class FilterSketch(GradSketch): - """Filter Gradient and Hessian outputs for the tree structure search using previously built trees""" + """Filter outputs for tree structure search using previously built trees. + + Outputs with zero Hessian mass are excluded. Zero total importance falls + back to a uniform distribution over the remaining outputs. If no output + has positive Hessian mass, raise ValueError instead of sampling NaNs. + """ def __init__(self, k=1, sample=True, smooth=0.1, ntrees=1): """ @@ -226,7 +231,7 @@ def before_iteration(self, build_info): # leaf values last_leaves = build_info['data']['train']['last_tree']['leaves'] self.queue.append(last_leaves) - self.max_nodes = max(int(last_leaves.max()), self.max_nodes) + self.max_nodes = max(int(last_leaves.max()) + 1, self.max_nodes) self.max_trees = max(last_leaves.shape[1], self.max_trees) def after_train(self, build_info): @@ -243,13 +248,12 @@ def after_train(self, build_info): def _calc_weights(self, grad, hess): if self.smooth >= 1: - return 0 + return cp.ones(grad.shape[1], dtype=cp.float64) if len(self.queue) == 0: - loss = (grad ** 2).sum(axis=0) - loss /= hess.sum(axis=0) - - return loss + loss = (grad ** 2).sum(axis=0, dtype=cp.float64) + denominator = hess.sum(axis=0, dtype=cp.float64) + return cp.where(denominator > 0, loss / cp.where(denominator > 0, denominator, 1), 0) grad_sum = cp.zeros((len(self.queue), self.max_trees, self.max_nodes, grad.shape[1],), dtype=cp.float32) @@ -261,26 +265,45 @@ def _calc_weights(self, grad, hess): grad_sum[n, i].scatter_add(prev_iter[:, i], grad) hess_sum[n, i].scatter_add(prev_iter[:, i], hess) - loss = (grad_sum ** 2 / (hess_sum + self.lambda_l2)).reshape((-1, grad.shape[1])) + denominator = hess_sum + self.lambda_l2 + loss = cp.where(denominator > 0, + grad_sum ** 2 / cp.where(denominator > 0, denominator, 1), 0) + loss = loss.reshape((-1, grad.shape[1])) - return loss.sum(axis=0) + return loss.sum(axis=0, dtype=cp.float64) def _select(self, pi): if self.sample: idx = cp.random.choice(cp.arange(pi.shape[0]), size=self.k, replace=True, p=pi) else: - idx = pi.argsort()[-self.k:] + candidates = cp.flatnonzero(pi > 0) + idx = candidates[pi[candidates].argsort()[-self.k:]] return idx def __call__(self, grad, hess): - pi = self._calc_weights(grad, hess) - pi = pi / pi.sum() + eligible = cp.broadcast_to(hess.sum(axis=0) > 0, (grad.shape[1],)) + if not bool(eligible.any()): + raise ValueError('FilterSketch cannot select an output: all Hessian sums are zero. ' + 'Check observed labels and positive sample weights.') + uniform = eligible.astype(cp.float64) + uniform /= uniform.sum() + + pi = cp.where(eligible, self._calc_weights(grad, hess), 0) + if not bool(cp.isfinite(pi).all()) or bool((pi < 0).any()): + raise ValueError('FilterSketch output importance must be finite and nonnegative') + scale = float(pi.max()) + if scale > 0: + pi /= scale + pi /= pi.sum() + else: + pi = uniform if self.smooth > 0: - pi = self.smooth * cp.ones_like(pi) / grad.shape[1] + (1 - self.smooth) * pi + smooth = min(self.smooth, 1) + pi = smooth * uniform + (1 - smooth) * pi idx = self._select(pi) diff --git a/tests/test_nan_losses.py b/tests/test_nan_losses.py index c5d2022..2e5eb1f 100644 --- a/tests/test_nan_losses.py +++ b/tests/test_nan_losses.py @@ -64,9 +64,9 @@ def test_without_missing_labels_matches_bce(losses, constant): ) -def test_missing_output_has_neutral_prior_and_zero_derivatives(losses): +def test_missing_labels_have_zero_derivatives(losses): cp = losses.cp - y = cp.asarray([[1, cp.nan, 0], [0, cp.nan, 0], [cp.nan, cp.nan, 1]], dtype=cp.float32) + y = cp.asarray([[1, 1, 0], [0, 0, 0], [cp.nan, cp.nan, 1]], dtype=cp.float32) loss = losses.BCEWithNaNLoss() expected = np.array([0, 0, np.log(0.5)], dtype=np.float32) np.testing.assert_allclose(loss.base_score(y).get(), expected, atol=1e-6) @@ -200,7 +200,6 @@ def test_shapes_and_aliases(losses): losses.BCEWithNaNMetricWeighted([1, 2])([0, 1], [.1, .9]) with pytest.raises(ValueError, match='one output weight'): losses.NaNGiniMetricAtN()([0, 1], [.1, .9], prod_weights=[1, 2]) - assert losses.BCEwithNaNMetricWeighted is losses.BCEWithNaNMetricWeighted assert isinstance(losses.loss_alias['bce_nan'], losses.BCEWithNaNLoss) assert isinstance(losses.loss_alias['multilabel_nan'], losses.BCEWithNaNLoss) assert type(losses.loss_alias['bce']) is losses.BCELoss @@ -211,7 +210,7 @@ def test_shapes_and_aliases(losses): for name in ['auc', 'roc', 'auc_nan']: assert isinstance(losses.BCEWithNaNLoss().get_metric_from_string(name), losses.NaNAucMetric) for name in ['BCEWithNaNLoss', 'NaNBCEMetric', 'NaNAucMetric', 'NaNAucMetricByClass', - 'NaNGiniMetricByClass', 'NaNGiniMetricAtN', 'BCEWithNaNMetricWeighted', 'BCEwithNaNMetricWeighted']: + 'NaNGiniMetricByClass', 'NaNGiniMetricAtN', 'BCEWithNaNMetricWeighted']: assert name in losses.__all__ @@ -225,7 +224,7 @@ def test_training_missing_targets(losses, sketch, metric): X = rng.normal(size=(128, 6)).astype(np.float32) y = (X[:, :3] > 0).astype(np.float32) y[rng.random(y.shape) < 0.25] = np.nan - y[:, 2] = np.nan # No labels in an entire output must not poison the ensemble. + y[96:, 2] = np.nan # Entirely missing evaluation outputs remain valid. model = GradientBoosting( 'bce_nan', metric=metric, ntrees=5, max_depth=2, min_data_in_leaf=2, max_bin=16, es=2, verbose=100, @@ -236,7 +235,6 @@ def test_training_missing_targets(losses, sketch, metric): assert prediction.shape == y[96:].shape assert np.isfinite(prediction).all() and np.isfinite(model.history).all() assert ((prediction > 0) & (prediction < 1)).all() - np.testing.assert_array_equal(prediction[:, 2], np.full(32, 0.5)) assert np.std(prediction[:, 0]) > 0 @@ -254,3 +252,249 @@ def test_cross_validation_missing_targets(losses): assert predictions.shape == y.shape and np.isfinite(predictions).all() assert np.isfinite(cv.predict(X)).all() assert len(cv.models) == 2 + + +@pytest.fixture +def training_data(): + rng = np.random.default_rng(67) + X = rng.normal(size=(128, 4)).astype(np.float32) + y = (X[:, :2] > 0).astype(np.float32) + return X, y + + +@pytest.mark.parametrize('with_eval', [False, True]) +def test_reject_unobserved_training_columns(losses, training_data, with_eval): + from py_boost import GradientBoosting + + X, y = training_data + y = np.tile(y, (1, 2)) + valid = [dict(X=X, y=y.copy())] if with_eval else None + y[:, [1, 3]] = np.nan + model = GradientBoosting('bce_nan', ntrees=2, max_bin=16, verbose=100) + with pytest.raises(ValueError, match=r'no observed labels.*\[1, 3\]'): + model.fit(X, y, eval_sets=valid) + + +@pytest.mark.parametrize('bad_label', [-1, 2, .5, np.inf, -np.inf]) +@pytest.mark.parametrize('with_eval', [False, True]) +def test_reject_invalid_training_labels(losses, training_data, bad_label, with_eval): + from py_boost import GradientBoosting + + X, y = training_data + valid = [dict(X=X, y=y.copy())] if with_eval else None + y[0, 0] = bad_label + weights = np.ones(len(y), dtype=np.float32) + weights[0] = 0 # Label validation applies even to zero-weight rows. + with pytest.raises(ValueError, match='Observed binary targets must be 0 or 1'): + GradientBoosting('bce_nan', ntrees=2, max_bin=16, verbose=100).fit( + X, y, sample_weight=weights, eval_sets=valid) + + +@pytest.mark.parametrize('bad_label', [-1, 2, .5, np.inf, -np.inf]) +def test_reject_invalid_evaluation_labels(losses, training_data, bad_label, monkeypatch): + from py_boost import GradientBoosting + + X, y = training_data + y_valid = y.copy() + y_valid[0, 1] = bad_label + loss = losses.BCEWithNaNLoss() + + def no_training(*args): + pytest.fail('Invalid evaluation labels must be rejected before computing gradients') + + monkeypatch.setattr(loss, 'get_grad_hess', no_training) + with pytest.raises(ValueError, match='Observed binary targets must be 0 or 1'): + GradientBoosting(loss, ntrees=2, max_bin=16, es=0, verbose=100).fit( + X, y, eval_sets=[dict(X=X, y=y_valid)]) + + +def test_targets_validated_once_before_training(losses, training_data): + from py_boost import GradientBoosting + + class CountingLoss(losses.BCEWithNaNLoss): + calls = 0 + + def preprocess_input(self, y_true): + self.calls += 1 + return super().preprocess_input(y_true) + + X, y = training_data + y[::4, 0] = np.nan + loss = CountingLoss() + model = GradientBoosting(loss, ntrees=3, max_depth=2, max_bin=16, es=0, verbose=100) + model.fit(X, y, eval_sets=[dict(X=X, y=y.copy())]) + assert loss.calls == 2 + assert len(model.models) == 3 + assert np.isnan(y[::4, 0]).all() + + +@pytest.mark.parametrize('es', [0, 2]) +@pytest.mark.parametrize('case', ['missing_labels', 'single_class_auc', 'zero_weights']) +def test_undefined_early_stopping_metric(losses, training_data, es, case): + from py_boost import GradientBoosting + + X, y = training_data + y_valid = y.copy() + metric = 'auc' if case == 'single_class_auc' else 'bce' + valid = dict(X=X, y=y_valid) + if case == 'missing_labels': + y_valid[:] = np.nan + elif case == 'single_class_auc': + y_valid[:] = 1 + else: + valid['sample_weight'] = np.zeros(len(y), dtype=np.float32) + model = GradientBoosting('bce_nan', metric=metric, ntrees=4, max_depth=2, + max_bin=16, es=es, verbose=100) + if es: + with pytest.raises(ValueError, match=r'Early stopping metric.*undefined.*evaluation set 0'): + model.fit(X, y, eval_sets=[valid]) + else: + model.fit(X, y, eval_sets=[valid]) + assert len(model.models) == 4 + assert np.isnan(model.history).all() + assert np.isfinite(model.predict(X)).all() + + +@pytest.mark.parametrize('undefined_last', [False, True]) +def test_only_selected_early_stopping_set_must_be_defined(losses, training_data, undefined_last): + from py_boost import GradientBoosting + + X, y = training_data + valid = [dict(X=X, y=np.full_like(y, np.nan)), dict(X=X, y=y)] + if undefined_last: + valid.reverse() + model = GradientBoosting('bce_nan', ntrees=3, max_depth=2, max_bin=16, es=2, verbose=100) + if undefined_last: + with pytest.raises(ValueError, match='undefined.*evaluation set 1'): + model.fit(X, y, eval_sets=valid) + else: + model.fit(X, y, eval_sets=valid) + assert np.isnan(np.array(model.history)[:, 0]).all() + assert np.isfinite(np.array(model.history)[:, 1]).all() + + +@pytest.mark.parametrize('scale', [1e-308, 1e-200, 1., 1e200, 1e307]) +def test_row_weight_scale_invariance(losses, data, scale): + y, p, w = data + for metric in [losses.NaNBCEMetric(), losses.BCEWithNaNMetricWeighted([1, 3, 2]), + losses.NaNAucMetric(), losses.NaNAucMetricByClass(), + losses.NaNGiniMetricByClass(), losses.NaNGiniMetricAtN()]: + np.testing.assert_allclose(metric(y, p, w * scale), metric(y, p, w), rtol=1e-12) + np.testing.assert_allclose( + losses.NaNBCEMetric()(y, p, w * scale, return_list=True).get(), + losses.NaNBCEMetric()(y, p, w, return_list=True).get(), rtol=1e-12, + ) + + +@pytest.mark.parametrize('weight', [1e308, 1e-308, 1e-320]) +def test_extreme_equal_row_weights(losses, weight): + y, p, w = [0, 1], [.1, .9], [weight, weight] + assert losses.NaNBCEMetric()(y, p, w) == pytest.approx(-np.log(.9)) + assert losses.NaNAucMetric()(y, p, w) == pytest.approx(1.) + cp = losses.cp + # The shared ROC AUC helper remains usable by the original binary metric. + assert losses.RocAucMetric()(cp.asarray(y)[:, None], cp.asarray(p)[:, None], cp.asarray(w)[:, None]) == 1. + + +def test_extreme_weights_with_missing_outputs_and_classes(losses): + y = [[0, np.nan], [1, np.nan], [np.nan, 0], [np.nan, 1]] + p = [[.1, np.nan], [.9, np.nan], [np.nan, .2], [np.nan, .8]] + w = [1e-300, 1e-300, 1e300, 1e300] + np.testing.assert_allclose(losses.NaNBCEMetric()(y, p, w, return_list=True).get(), + [-np.log(.9), -np.log(.8)]) + np.testing.assert_allclose(losses.NaNAucMetricByClass()(y, p, w), [1., 1.]) + assert losses.NaNAucMetric()([0, 1], [.1, .9], [1e-300, 1e300]) == 1. + assert losses.NaNAucMetric()([0, 1], [.9, .1], [1e-300, 1e300]) == 0. + assert np.isinf(losses.NaNBCEMetric()([1, 0], [0, .1], [1e-300, 1e300])) + + +@pytest.mark.parametrize('sample', [False, True]) +@pytest.mark.parametrize('smooth', [0., .1, 1.]) +def test_filter_sketch_ignores_zero_weight_training_outputs(losses, training_data, sample, smooth): + from py_boost import GradientBoosting + from py_boost.multioutput.sketching import FilterSketch + + class RecordingSketch(FilterSketch): + def _select(self, pi): + probabilities.append(pi.get()) + return super()._select(pi) + + X, y = training_data + weights = (np.arange(len(y)) % 2).astype(np.float32) + y[weights > 0, 1] = np.nan + probabilities = [] + model = GradientBoosting('bce_nan', ntrees=3, max_depth=2, max_bin=16, es=0, verbose=100, + use_hess=True, multioutput_sketch=RecordingSketch(k=2, sample=sample, smooth=smooth)) + model.fit(X, y, sample_weight=weights, eval_sets=[dict(X=X, y=y)]) + np.testing.assert_allclose(probabilities, [[1., 0.]] * 3) + assert np.isfinite(model.predict(X)).all() + assert np.isfinite(model.history).all() + + +def test_filter_sketch_rejects_all_zero_training_weights(losses, training_data): + from py_boost import GradientBoosting + from py_boost.multioutput.sketching import FilterSketch + + X, y = training_data + model = GradientBoosting('bce_nan', ntrees=2, max_bin=16, verbose=100, + use_hess=True, multioutput_sketch=FilterSketch()) + with pytest.raises(ValueError, match='FilterSketch cannot select.*all Hessian sums are zero'): + model.fit(X, y, sample_weight=np.zeros(len(y), dtype=np.float32)) + + +@pytest.mark.parametrize('queued', [False, True]) +@pytest.mark.parametrize('shared_hessian', [False, True]) +@pytest.mark.parametrize('sample', [False, True]) +def test_filter_sketch_zero_importance_fallback(losses, queued, shared_hessian, sample): + from types import SimpleNamespace + from py_boost.multioutput.sketching import FilterSketch + + cp = losses.cp + grad = cp.zeros((4, 3), dtype=cp.float32) + hess = cp.ones((4, 1), dtype=cp.float32) if shared_hessian else cp.asarray([[1., 0., 1.]] * 4) + sketch = FilterSketch(k=2, sample=sample, smooth=0.) + info = dict(data=dict(train=dict(features_gpu=grad, last_tree=dict(leaves=cp.zeros((4, 1), dtype=cp.int32)))), + builder=SimpleNamespace(params=dict(lambda_l2=0.)), num_iter=1) + sketch.before_train(info) + if queued: + sketch.before_iteration(info) + assert sketch.max_nodes == 1 # A tree with one leaf still needs one accumulator. + probabilities = [] + select = sketch._select + + def record(pi): + probabilities.append(pi.get()) + return select(pi) + + sketch._select = record + g, h = sketch(grad, hess) + expected = [1 / 3] * 3 if shared_hessian else [.5, 0., .5] + np.testing.assert_allclose(probabilities, [expected]) + assert bool(cp.isfinite(g).all()) and bool((h > 0).all()) + + +def test_filter_sketch_keeps_distinct_leaf_accumulators(losses): + from types import SimpleNamespace + from py_boost.multioutput.sketching import FilterSketch + + cp = losses.cp + grad = cp.asarray([[1., 2.], [3., 4.]], dtype=cp.float32) + hess = cp.ones_like(grad) + sketch = FilterSketch(smooth=0.) + info = dict(data=dict(train=dict(features_gpu=grad, last_tree=dict(leaves=cp.asarray([[0], [1]])))), + builder=SimpleNamespace(params=dict(lambda_l2=1.)), num_iter=1) + sketch.before_train(info) + sketch.before_iteration(info) + np.testing.assert_allclose(sketch._calc_weights(grad, hess).get(), [5., 10.]) + + +def test_cross_validation_rejects_unobserved_training_fold(losses, training_data): + from py_boost import GradientBoosting + from py_boost.cv import CrossValidation + + X, y = training_data + folds = np.arange(len(y)) % 2 + y[folds == 1, 1] = np.nan + model = GradientBoosting('bce_nan', ntrees=2, max_depth=2, max_bin=16, verbose=100) + with pytest.raises(ValueError, match=r'no observed labels.*\[1\]'): + CrossValidation(model).fit_predict(X, y, cv=folds) diff --git a/tutorials/Tutorial_4_Handle_null_targets.ipynb b/tutorials/Tutorial_4_Handle_null_targets.ipynb index aa74b76..b4956f7 100644 --- a/tutorials/Tutorial_4_Handle_null_targets.ipynb +++ b/tutorials/Tutorial_4_Handle_null_targets.ipynb @@ -16,201 +16,139 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", - "# Optional: set the device to run\n", - "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n", - "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n", + "# Optional: select a CUDA device before importing Py-Boost.\n", + "# os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n", "\n", - "os.makedirs('../data', exist_ok=True)\n", "import numpy as np\n", - "import joblib\n", "from sklearn.datasets import make_regression\n", "\n", - "# simple case - just one class is used\n", "from py_boost import GradientBoosting\n", - "from py_boost.multioutput.sketching import *" + "from py_boost.multioutput.sketching import RandomProjectionSketch\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Generate dummy multilabel task with NaN values in target\n", + "### Generate a multilabel task with missing targets\n", "\n", - "Some times it happends that some target values in multioutput task are missing. For example, you are solving multilabel task and some labels are unknown for some of the rows, so acually your target could be one of 0/1/NaN. Normaly you can not using ML algorithms directly in that case, so you can do one of the following:\n", + "A binary target has three possible values: `1` is an observed positive, `0` is an observed negative, and `NaN` is an unknown label. Replacing unknown labels with `0` incorrectly turns them into negative examples.\n", "\n", - "- Drop NaN rows, but that case you are going to miss some part of the data\n", - "- Train binary models separately, but your model will be more complex and probably overfitted\n", - "- Fill NaNs with 0 or 1, so your labeling will become wrong\n", - "- Use Neural Networks with masked loss function\n", + "Py-Boost provides `BCEWithNaNLoss` and NaN-aware metrics. Missing labels contribute no gradients or Hessians, while other observed labels on the same row remain useful for training.\n", "\n", - "In Py-Boost you can write the loss wrapper to handle such scenario and train your model directly on known labels ignoring NaNs, and here is shown how.\n", - "\n", - "We will create it as the regression task and then thresholding the target. And then add some random NaNs" + "Here we generate a regression task, threshold each target into a binary label, and randomly mark half the labels as unknown.\n" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 2.33 s, sys: 1.66 s, total: 3.99 s\n", - "Wall time: 876 ms\n" - ] - } - ], + "outputs": [], "source": [ "%%time\n", "X, y = make_regression(150000, 100, n_targets=10, random_state=42)\n", - "# binarize\n", "y = (y > y.mean(axis=0)).astype(np.float32)\n", - "# add some NaNs\n", - "y[np.random.rand(150000, 10) > 0.5] = np.nan\n", - "\n", + "rng = np.random.default_rng(42)\n", + "y[rng.random(y.shape) > 0.5] = np.nan\n", "\n", "X_test, y_test = X[:50000], y[:50000]\n", - "X, y = X[-50000:], y[-50000:]" + "X, y = X[-50000:], y[-50000:]\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### NaN loss and metric wrappers\n", + "### Use the built-in loss and metrics\n", "\n", - "Here it is shown how to write loss wrapper ignoring NaNs" + "Every training target column must contain at least one observed label. A fully unknown training column raises `ValueError` listing its zero-based index; it is not silently removed or assigned a probability of 0.5. Both training and evaluation targets are checked for values other than `0`, `1`, and `NaN` before training.\n", + "\n", + "Use positive `lambda_l2` so leaves without observed labels have finite updates. The loss also accepts the string aliases `bce_nan` and `multilabel_nan`; its default metric is `NaNBCEMetric`.\n" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "import cupy as cp\n", - "from py_boost.gpu.losses import BCELoss\n", - "\n", - "class BCEWithNaNLoss(BCELoss):\n", - " \n", - " def base_score(self, y_true):\n", - " # Replace .mean with nanmean function to calc base score\n", - " means = cp.clip(cp.nanmean(y_true, axis=0), self.clip_value, 1 - self.clip_value)\n", - " return cp.log(means / (1 - means))\n", - " \n", - " def get_grad_hess(self, y_true, y_pred):\n", - " # first, get nan mask for y_true\n", - " mask = cp.isnan(y_true)\n", - " # then, compute loss with any values at nan places just to prevent the exception\n", - " grad, hess = super().get_grad_hess(cp.where(mask, 0, y_true), y_pred)\n", - " # invert mask\n", - " mask = (~mask).astype(cp.float32)\n", - " # multiply grad and hess on inverted mask\n", - " # now grad and hess eq. 0 on NaN points\n", - " # that actually means that prediction on that place should not be updated\n", - " grad = grad * mask\n", - " hess = hess * mask\n", - " \n", - " return grad, hess\n" + "from py_boost.gpu.losses import (\n", + " BCEWithNaNLoss,\n", + " NaNAucMetric,\n", + " NaNAucMetricByClass,\n", + ")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "And here is column-wise roc-auc metric ignoring NaNs" + "### Undefined evaluation outputs\n", + "\n", + "An output can have training labels but no observed labels in validation. This is allowed: per-output metrics return `NaN` for undefined outputs, and macro metrics exclude them. ROC AUC also requires positive row weight in both classes; BCE only requires at least one positively weighted observed label.\n", + "\n", + "To demonstrate this, hide the last validation target while retaining its training labels. If every output is undefined for the scalar metric used by early stopping, training raises an explicit error. Provide a suitable evaluation set or metric, or use `es=0` to train without early stopping and retain `NaN` in the metric history.\n" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "from py_boost.gpu.losses.metrics import Metric, auc\n", - "\n", - "class NaNAucMetric(Metric):\n", - " \n", - " def __call__(self, y_true, y_pred, sample_weight=None):\n", - " \n", - " aucs = []\n", - " mask = ~cp.isnan(y_true)\n", - " \n", - " for i in range(y_true.shape[1]):\n", - " m = mask[:, i]\n", - " w = None if sample_weight is None else sample_weight[:, 0][m]\n", - " aucs.append(\n", - " auc(y_true[:, i][m], y_pred[:, i][m], w)\n", - " )\n", - " \n", - " return np.mean(aucs)\n", - " \n", - " def compare(self, v0 ,v1):\n", - "\n", - " return v0 > v1 \n", - " \n", - " \n", - " " + "y_test = y_test.copy()\n", + "y_test[:, -1] = np.nan\n" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[08:31:38] Stdout logging level is INFO.\n", - "[08:31:38] GDBT train starts. Max iter 1000, early stopping rounds 200\n", - "[08:31:39] Iter 0; Sample 0, score = 0.7906884535541213; \n", - "[08:31:41] Iter 100; Sample 0, score = 0.9687261163054176; \n", - "[08:31:44] Iter 200; Sample 0, score = 0.9785187659166686; \n", - "[08:31:46] Iter 300; Sample 0, score = 0.9844858052685057; \n", - "[08:31:49] Iter 400; Sample 0, score = 0.9883780152591723; \n", - "[08:31:51] Iter 500; Sample 0, score = 0.9908004122540589; \n", - "[08:31:54] Iter 600; Sample 0, score = 0.9923353340683694; \n", - "[08:31:57] Iter 700; Sample 0, score = 0.9935137491384962; \n", - "[08:31:59] Iter 800; Sample 0, score = 0.9943018456130359; \n", - "[08:32:02] Iter 900; Sample 0, score = 0.9949417958344802; \n", - "[08:32:04] Iter 999; Sample 0, score = 0.9954331107999328; \n", - "CPU times: user 32.1 s, sys: 1.59 s, total: 33.7 s\n", - "Wall time: 31.9 s\n" - ] - }, - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "%%time\n", - "model = GradientBoosting(BCEWithNaNLoss(), NaNAucMetric(), lr=0.01,\n", - " verbose=100, ntrees=1000, es=200, multioutput_sketch=RandomProjectionSketch(1))\n", + "model = GradientBoosting(\n", + " BCEWithNaNLoss(), metric=NaNAucMetric(), lr=0.01, lambda_l2=1,\n", + " verbose=100, ntrees=1000, es=200,\n", + " multioutput_sketch=RandomProjectionSketch(1),\n", + ")\n", + "model.fit(X, y, eval_sets=[{'X': X_test, 'y': y_test}])\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Report per-output quality\n", "\n", - "model.fit(X, y, eval_sets=[{'X': X_test, 'y': y_test},])" + "The last AUC is `NaN` because its validation labels are unknown. Predictions for that output are still learned from the observed training labels. Per-output metrics are for reporting; use a scalar metric for early stopping." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "execution_count": null, + "outputs": [], + "source": [ + "prediction = model.predict(X_test)\n", + "validation_auc = NaNAucMetricByClass()(y_test, prediction)\n", + "print(validation_auc)\n", + "assert np.isnan(validation_auc[-1])\n", + "assert np.isfinite(validation_auc[:-1]).all()\n", + "assert np.isfinite(prediction).all()" ] } ], "metadata": { "kernelspec": { - "display_name": "rapids-22.06", + "display_name": "Python 3", "language": "python", - "name": "rapids-22.06" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -221,8 +159,7 @@ "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" + "pygments_lexer": "ipython3" } }, "nbformat": 4, From 84f692f48af5c1064bf603c3fd18290265bfc9d4 Mon Sep 17 00:00:00 2001 From: Rinchin Date: Wed, 28 Oct 2026 21:38:41 +0300 Subject: [PATCH 3/5] Enforce weighted NaN-target training policy and preserve numeric precision --- README.md | 42 ++- py_boost/callbacks/callback.py | 6 +- py_boost/gpu/boosting.py | 29 +- py_boost/gpu/losses/losses.py | 76 ++++- py_boost/gpu/losses/metrics.py | 42 ++- py_boost/multioutput/sketching.py | 8 +- tests/test_nan_losses.py | 11 +- tests/test_nan_training_contract.py | 320 ++++++++++++++++++ .../Tutorial_4_Handle_null_targets.ipynb | 10 +- 9 files changed, 482 insertions(+), 62 deletions(-) create mode 100644 tests/test_nan_training_contract.py diff --git a/README.md b/README.md index 12d08c2..bc501db 100644 --- a/README.md +++ b/README.md @@ -72,11 +72,15 @@ More examples are coming soon. ## Missing binary targets For binary or multilabel targets, use `NaN` for unknown labels and `0` for -observed negatives. `BCEWithNaNLoss` excludes missing labels from gradients, -Hessians and initial class frequencies. Every training target column must -contain at least one observed label. Otherwise, training raises `ValueError` -listing the zero-based indices of the affected columns; no columns are -silently removed. Entirely unobserved columns are allowed in evaluation sets. +observed negatives. `BCEWithNaNLoss` excludes missing labels from gradients +and Hessians. Its initial probabilities are weighted frequencies over observed +labels; zero-weight rows do not affect them. Every training target column must +contain an observed label with positive row weight. Without `sample_weight`, +each row has weight 1. Both all-NaN columns and columns whose observed labels +all have zero weight raise `ValueError` listing their zero-based indices before +initialization or tree construction. This check applies to the training +partition of each cross-validation fold; no columns are silently removed. +Entirely unobserved columns are allowed in evaluation sets. Keep `lambda_l2 > 0` so leaves without observed labels have finite updates. ```python @@ -121,8 +125,10 @@ labels, or set `es=0` to keep training and record NaN in the evaluation history. The per-class vector metrics are for reporting only. Observed targets must be `0` or `1`; `NaN` is the only missing-label marker. -Training and evaluation labels are validated once before training, including -labels on zero-weight rows. Predictions must be finite wherever a label +Training and evaluation labels are validated once in their original dtype +before conversion to float32, including labels on zero-weight rows. A float64 +label such as `1.00000001` is rejected instead of being rounded to `1`. +Predictions must be finite wherever a label has positive row weight; invalid predictions are reported as errors rather than treated as missing labels. BCE additionally requires probabilities in `[0, 1]`, while ROC AUC/Gini also accept finite decision scores. @@ -138,9 +144,21 @@ requires an explicit scorer for missing labels. Outputs with zero total Hessian are excluded from selection. If all remaining output importances are zero, selection uses a uniform distribution over the eligible outputs; if none remain, it raises an error identifying the lack of -positive Hessian mass. The training-column check counts observed labels before -row weights, so zero-weight observations alone do not make an output eligible -for `FilterSketch` with `use_hess=True`. - -GPU regression tests can be run with `python -m pytest -q tests/test_nan_losses.py` +positive Hessian mass. The separate training-column check requires an observed +label with positive row weight in each training output. Importance squaring and +accumulation use float64 to avoid float32 overflow and underflow. Zero gradients +at an optimum are valid and do not trigger a missing-training-label error. + +For `BCEWithNaNLoss`, row weights are validated as finite and nonnegative +before conversion. Training weights are stored as float32: overflow to infinity +or conversion of a positive weight to zero raises an explicit representation +error with the affected row indices. Training weights are not normalized, so +their scale retains its effect on `lambda_l2`. Evaluation weights stay float64 +through `fit`, allowing metrics to handle weights such as `1e308` and `1e-308`. +All-zero training weights are rejected by the per-output check; all-zero +evaluation weights are allowed for reporting and produce an undefined metric. +These strict validation and weighted-initialization policies apply only to +`BCEWithNaNLoss`; existing losses retain their preprocessing and initial scores. + +GPU regression tests can be run with `python -m pytest -q tests` after installing Py-Boost, a compatible CuPy build, and pytest. diff --git a/py_boost/callbacks/callback.py b/py_boost/callbacks/callback.py index 66c7c98..6520d61 100644 --- a/py_boost/callbacks/callback.py +++ b/py_boost/callbacks/callback.py @@ -218,7 +218,11 @@ def set_verbosity_level(verbose): class EarlyStopping(Callback): - """Callback for early stopping""" + """Stop using the last validation set's scalar metric. + + With early stopping enabled, an undefined (NaN) score raises ValueError. + Set num_rounds=0 to disable early stopping and this check. + """ def __init__(self, num_rounds=100): diff --git a/py_boost/gpu/boosting.py b/py_boost/gpu/boosting.py index 0b5d2bb..5548dee 100644 --- a/py_boost/gpu/boosting.py +++ b/py_boost/gpu/boosting.py @@ -5,7 +5,7 @@ except Exception: pass from .base import Ensemble -from .losses import loss_alias +from .losses import Loss, loss_alias from .tree import DepthwiseTreeBuilder from .utils import pad_and_move, validate_input from ..callbacks.callback import EarlyStopping, EvalHistory, CallbackPipeline @@ -282,19 +282,24 @@ def _create_build_info(self, mempool, X, X_enc, y, sample_weight, max_bin, borde Returns: DepthwiseTreeBuilder, build_info """ - # quantization + cp.random.seed(self.seed) - y = cp.array(y, order='C', dtype=cp.float32) + def prepare_target(target, weights, training): + prepare = getattr(self.loss, 'prepare_input', None) + if prepare is None: + # Keep custom losses implementing the original interface working. + return Loss.prepare_input(self.loss, target, weights, training=training) + return prepare(target, weights, training=training) - if sample_weight is not None: - sample_weight = cp.array(sample_weight, order='C', dtype=cp.float32) + y, sample_weight = prepare_target(y, sample_weight, training=True) + valid = [prepare_target(x['y'], x['sample_weight'], training=False) for x in eval_sets] + y_val = [target for target, _ in valid] + w_val = [weights for _, weights in valid] + initialize = getattr(self.loss, 'initial_score', None) + self.base_score = self.loss.base_score(y) if initialize is None else initialize(y, sample_weight) X_cp = pad_and_move(X_enc) - X_val = [cp.array(x, order='C') for x in eval_enc] - y_val = [cp.array(x['y'], order='C', dtype=cp.float32) for x in eval_sets] - w_val = [None if x['sample_weight'] is None else cp.array(x['sample_weight'], order='C', dtype=cp.float32) - for x in eval_sets] # save nfeatures for the feature importances self.nfeats = X.shape[1] @@ -313,12 +318,6 @@ def _create_build_info(self, mempool, X, X_enc, y, sample_weight, max_bin, borde max_depth=self.max_depth, max_bin=max_bin, ) - cp.random.seed(self.seed) - - y = self.loss.preprocess_input(y) - y_val = [self.loss.preprocess_input(x) for x in y_val] - self.base_score = self.loss.base_score(y) - # init ensembles ens = cp.empty((y.shape[0], self.base_score.shape[0]), order='C', dtype=cp.float32) ens[:] = self.base_score diff --git a/py_boost/gpu/losses/losses.py b/py_boost/gpu/losses/losses.py index d9b02d9..7f71896 100644 --- a/py_boost/gpu/losses/losses.py +++ b/py_boost/gpu/losses/losses.py @@ -8,6 +8,7 @@ CUDA_FOUND = False from .metrics import metric_alias, RMSEMetric, RMSLEMetric, BCEMetric, NaNBCEMetric, NaNAucMetric +from .metrics import _row_weights, _rescale_row_weights from .multiclass_metrics import multiclass_metric_alias, CrossEntropyMetric @@ -47,6 +48,21 @@ def preprocess_input(self, y_true): """ return y_true + def prepare_input(self, y_true, sample_weight=None, training=False): + """Prepare a dataset using the existing float32 preprocessing contract. + + Losses may override this to validate values before conversion or + retain higher precision for evaluation weights. + """ + y_true = cp.array(y_true, order='C', dtype=cp.float32) + if sample_weight is not None: + sample_weight = cp.array(sample_weight, order='C', dtype=cp.float32) + return self.preprocess_input(y_true), sample_weight + + def initial_score(self, y_true, sample_weight=None): + """Initialize while preserving existing base_score(y_true) overrides.""" + return self.base_score(y_true) + def postprocess_output(self, y_pred): """Method defines how to postprocess sum of trees to output prediction (ex. apply sigmoid for BCELoss) @@ -158,25 +174,63 @@ class BCEWithNaNLoss(BCELoss): """Binary/multilabel log loss with missing (NaN) targets. Missing labels contribute zero gradient and Hessian. Initial log odds - use only observed labels. Every training output must have at least one - observed label; entirely missing evaluation outputs are allowed. Use - positive lambda_l2 for leaves with no observed labels. The default - evaluation metric also ignores missing labels. + use weighted observed-label frequencies. Every training output must have + an observed label with positive row weight. Evaluation outputs may be + entirely missing. Use positive lambda_l2 for leaves with no observed + labels. The default evaluation metric also ignores missing labels. """ def preprocess_input(self, y_true): - if bool((~cp.isnan(y_true) & (y_true != 0) & (y_true != 1)).any()): + observed = ~cp.isnan(y_true) + if bool((observed & (y_true != 0) & (y_true != 1)).any()): raise ValueError('Observed binary targets must be 0 or 1; use NaN for missing labels') return y_true - def base_score(self, y_true): + def prepare_input(self, y_true, sample_weight=None, training=False): + # Validate labels in their original dtype, before rounding to float32. + y_true = self.preprocess_input(cp.asarray(y_true)) + if sample_weight is not None: + sample_weight = _row_weights(sample_weight, y_true.shape[0]) + if training: + converted = sample_weight.astype(cp.float32) + invalid = ~cp.isfinite(converted) | ((sample_weight > 0) & (converted == 0)) + if bool(invalid.any()): + rows = cp.flatnonzero(invalid[:, 0]).get().tolist() + raise ValueError( + 'Training sample_weight cannot be represented in float32 ' + 'without overflow or loss of positive weights; affected rows: ' + f'{rows}' + ) + sample_weight = converted + # Evaluation weights stay float64 for stable metric aggregation. + sample_weight = cp.array(sample_weight, order='C') + return cp.array(y_true, order='C', dtype=cp.float32), sample_weight + + def initial_score(self, y_true, sample_weight=None): + return self.base_score(y_true, sample_weight) + + def base_score(self, y_true, sample_weight=None): + y_true = cp.asarray(y_true) + if y_true.ndim == 1: + y_true = y_true[:, None] observed = ~cp.isnan(y_true) + if sample_weight is not None: + weights = _row_weights(sample_weight, y_true.shape[0]) + observed &= weights > 0 counts = observed.sum(axis=0) - missing_outputs = cp.flatnonzero(counts == 0).get().tolist() - if missing_outputs: - raise ValueError('Training target columns have no observed labels ' - f'(zero-based indices): {missing_outputs}') - means = cp.where(observed, y_true, 0).sum(axis=0) / counts.astype(y_true.dtype) + missing_outputs = cp.flatnonzero(counts == 0) + if missing_outputs.size: + raise ValueError( + 'Training target columns have no observed labels ' + f'(zero-based indices): {missing_outputs.get().tolist()} ' + '(positive sample weight required)' + ) + if sample_weight is None: + means = cp.where(observed, y_true, 0).sum(axis=0) / counts + else: + weights = _rescale_row_weights(cp.where(observed, weights, 0)) + means = (cp.where(observed, y_true, 0) * weights).sum(axis=0) / weights.sum(axis=0) + means = means.astype(cp.result_type(y_true.dtype, cp.float32)) means = cp.clip(means, self.clip_value, 1 - self.clip_value) return cp.log(means / (1 - means)) diff --git a/py_boost/gpu/losses/metrics.py b/py_boost/gpu/losses/metrics.py index fd9ec5e..4c7e74d 100644 --- a/py_boost/gpu/losses/metrics.py +++ b/py_boost/gpu/losses/metrics.py @@ -208,6 +208,28 @@ def compare(self, v0, v1): return v0 > v1 +def _row_weights(sample_weight, n_samples): + """Validate row weights without losing their original floating-point range.""" + if sample_weight is None: + return cp.ones((n_samples, 1), dtype=cp.float64) + weights = cp.asarray(sample_weight, dtype=cp.float64) + if weights.ndim == 1: + weights = weights[:, None] + if weights.shape != (n_samples, 1): + raise ValueError('sample_weight must have shape (n_samples,) or (n_samples, 1)') + if not bool(cp.isfinite(weights).all()) or bool((weights < 0).any()): + raise ValueError('sample_weight must be finite and nonnegative') + return weights + + +def _rescale_row_weights(weights): + """Rescale each output independently after excluding its missing labels.""" + if weights.shape[0] == 0: + return weights + scale = weights.max(axis=0, keepdims=True) + return weights / cp.where(scale > 0, scale, 1) + + def _nan_metric_inputs(y_true, y_pred, sample_weight): """Validate shapes and prepare row weights for metrics with missing labels.""" y_true, y_pred = cp.asarray(y_true), cp.asarray(y_pred) @@ -218,16 +240,7 @@ def _nan_metric_inputs(y_true, y_pred, sample_weight): if y_true.ndim != 2 or y_true.shape != y_pred.shape or y_true.shape[1] == 0: raise ValueError('Targets and predictions must have the same nonempty output dimension') - if sample_weight is None: - weights = cp.ones((y_true.shape[0], 1), dtype=cp.float64) - else: - weights = cp.asarray(sample_weight, dtype=cp.float64) - if weights.ndim == 1: - weights = weights[:, None] - if weights.shape != (y_true.shape[0], 1): - raise ValueError('sample_weight must have shape (n_samples,) or (n_samples, 1)') - if not bool(cp.isfinite(weights).all()) or bool((weights < 0).any()): - raise ValueError('sample_weight must be finite and nonnegative') + weights = _row_weights(sample_weight, y_true.shape[0]) observed = ~cp.isnan(y_true) & (weights > 0) if bool((observed & (y_true != 0) & (y_true != 1)).any()): raise ValueError('Observed binary targets must be 0 or 1') @@ -253,9 +266,16 @@ def _nan_weighted_mean(values, weights=None): valid = ~np.isnan(values) & (weights > 0) if not valid.any(): return float('nan') + selected = values[valid] + # Positive original weights retain infinite errors even if their + # normalized values would underflow to zero. + if np.isposinf(selected).any(): + return float('nan') if np.isneginf(selected).any() else float('inf') + if np.isneginf(selected).any(): + return float('-inf') # Rescale first to avoid overflow for large finite output weights. weights = weights[valid] / weights[valid].max() - return float(np.sum(values[valid] * weights) / weights.sum()) + return float(np.sum(selected * weights) / weights.sum()) class NaNBCEMetric(BCEMetric): diff --git a/py_boost/multioutput/sketching.py b/py_boost/multioutput/sketching.py index 01761bb..25b889a 100644 --- a/py_boost/multioutput/sketching.py +++ b/py_boost/multioutput/sketching.py @@ -252,14 +252,14 @@ def _calc_weights(self, grad, hess): return cp.ones(grad.shape[1], dtype=cp.float64) if len(self.queue) == 0: - loss = (grad ** 2).sum(axis=0, dtype=cp.float64) + loss = cp.square(grad, dtype=cp.float64).sum(axis=0) denominator = hess.sum(axis=0, dtype=cp.float64) return cp.where(denominator > 0, loss / cp.where(denominator > 0, denominator, 1), 0) grad_sum = cp.zeros((len(self.queue), self.max_trees, self.max_nodes, - grad.shape[1],), dtype=cp.float32) + grad.shape[1],), dtype=cp.float64) hess_sum = cp.zeros((len(self.queue), self.max_trees, self.max_nodes, - hess.shape[1],), dtype=cp.float32) + hess.shape[1],), dtype=cp.float64) for n, prev_iter in enumerate(self.queue): for i in range(prev_iter.shape[1]): @@ -285,7 +285,7 @@ def _select(self, pi): def __call__(self, grad, hess): - eligible = cp.broadcast_to(hess.sum(axis=0) > 0, (grad.shape[1],)) + eligible = cp.broadcast_to(hess.sum(axis=0, dtype=cp.float64) > 0, (grad.shape[1],)) if not bool(eligible.any()): raise ValueError('FilterSketch cannot select an output: all Hessian sums are zero. ' 'Check observed labels and positive sample weights.') diff --git a/tests/test_nan_losses.py b/tests/test_nan_losses.py index 2e5eb1f..54d0f81 100644 --- a/tests/test_nan_losses.py +++ b/tests/test_nan_losses.py @@ -410,7 +410,7 @@ def test_extreme_weights_with_missing_outputs_and_classes(losses): @pytest.mark.parametrize('sample', [False, True]) @pytest.mark.parametrize('smooth', [0., .1, 1.]) -def test_filter_sketch_ignores_zero_weight_training_outputs(losses, training_data, sample, smooth): +def test_zero_weight_training_outputs_rejected_before_sketch(losses, training_data, sample, smooth): from py_boost import GradientBoosting from py_boost.multioutput.sketching import FilterSketch @@ -425,10 +425,9 @@ def _select(self, pi): probabilities = [] model = GradientBoosting('bce_nan', ntrees=3, max_depth=2, max_bin=16, es=0, verbose=100, use_hess=True, multioutput_sketch=RecordingSketch(k=2, sample=sample, smooth=smooth)) - model.fit(X, y, sample_weight=weights, eval_sets=[dict(X=X, y=y)]) - np.testing.assert_allclose(probabilities, [[1., 0.]] * 3) - assert np.isfinite(model.predict(X)).all() - assert np.isfinite(model.history).all() + with pytest.raises(ValueError, match=r'no observed labels.*\[1\]'): + model.fit(X, y, sample_weight=weights, eval_sets=[dict(X=X, y=y)]) + assert probabilities == [] def test_filter_sketch_rejects_all_zero_training_weights(losses, training_data): @@ -438,7 +437,7 @@ def test_filter_sketch_rejects_all_zero_training_weights(losses, training_data): X, y = training_data model = GradientBoosting('bce_nan', ntrees=2, max_bin=16, verbose=100, use_hess=True, multioutput_sketch=FilterSketch()) - with pytest.raises(ValueError, match='FilterSketch cannot select.*all Hessian sums are zero'): + with pytest.raises(ValueError, match=r'no observed labels.*\[0, 1\]'): model.fit(X, y, sample_weight=np.zeros(len(y), dtype=np.float32)) diff --git a/tests/test_nan_training_contract.py b/tests/test_nan_training_contract.py new file mode 100644 index 0000000..d731208 --- /dev/null +++ b/tests/test_nan_training_contract.py @@ -0,0 +1,320 @@ +"""CUDA regressions for weighted NaN-aware training and shared loss interfaces.""" +import numpy as np +import pytest + + +@pytest.fixture(scope='module') +def gpu(): + cp = pytest.importorskip('cupy') + try: + count = cp.cuda.runtime.getDeviceCount() + except cp.cuda.runtime.CUDARuntimeError as exc: + pytest.skip(f'CUDA device unavailable: {exc}') + if count == 0: + pytest.skip('CUDA required') + return cp + + +@pytest.fixture +def data(): + X = np.random.default_rng(12).normal(size=(64, 6)).astype(np.float32) + return X, (X[:, :2] > 0).astype(np.float64) + + +def make_model(**kwargs): + from py_boost import GradientBoosting + params = dict(loss='bce_nan', ntrees=1, max_depth=2, max_bin=16, es=0, verbose=100) + params.update(kwargs) + return GradientBoosting(**params) + + +@pytest.mark.parametrize('sketch', [None, 'filter']) +def test_strict_rejects_zero_effective_labels(gpu, data, sketch): + from py_boost.multioutput.sketching import FilterSketch + X, y = data + y = np.column_stack([y, np.full(len(y), np.nan)]) + y[:2, 2] = [0, 1] + weights = np.ones(len(y)) + weights[:2] = 0 + model = make_model(multioutput_sketch=FilterSketch(1) if sketch else None, use_hess=True) + with pytest.raises(ValueError) as exc: + model.fit(X, y, sample_weight=weights) + assert '2' in str(exc.value) + + +@pytest.mark.parametrize('positive_weight, expected', [(9., .9), (0., 1e-7)]) +def test_weighted_initial_frequency(gpu, data, positive_weight, expected): + X, _ = data + y = np.repeat([0., 1.], len(X) // 2)[:, None] + weights = np.where(y[:, 0] == 1, positive_weight, 1.) + model = make_model().fit(X, y, sample_weight=weights) + initial = model.loss.postprocess_output(model.base_score) + np.testing.assert_allclose(initial, [expected], rtol=1e-5, atol=1e-8) + + +@pytest.mark.parametrize('where', ['training', 'evaluation']) +def test_validate_labels_before_float32_conversion(gpu, data, where): + X, y = data + invalid = y.copy() + invalid[0, 0] = 1.00000001 + with pytest.raises(ValueError, match='binary targets'): + if where == 'training': + make_model().fit(X, invalid) + else: + make_model().fit(X, y, eval_sets=[{'X': X, 'y': invalid}]) + + +@pytest.mark.parametrize('weight', [-1., np.nan, np.inf, 1e-308, 1e308]) +@pytest.mark.parametrize('evaluate', [False, True]) +def test_training_weight_validation_and_representation(gpu, data, weight, evaluate): + X, y = data + with pytest.raises(ValueError, match='(?i)(weight|finite|nonnegative|represent|float32)'): + make_model().fit(X, y, sample_weight=np.full(len(y), weight), + eval_sets=[{'X': X, 'y': y}] if evaluate else None) + + +@pytest.mark.parametrize('weight', [1e-308, 1e308]) +@pytest.mark.parametrize('metric', [None, 'auc', 'gini_nan']) +def test_extreme_evaluation_weights_through_fit(gpu, data, weight, metric): + X, y = data + model = make_model(es=2, metric=metric) + model.fit(X, y, eval_sets=[{'X': X, 'y': y, 'sample_weight': np.full(len(y), weight)}]) + assert np.isfinite(model.history).all() + + +@pytest.mark.parametrize('stage', ['initial', 'queued']) +@pytest.mark.parametrize('factor', [1., 1e-30, 1e20]) +def test_filter_importance_precision(gpu, stage, factor): + from py_boost.multioutput.sketching import FilterSketch + cp = gpu + sketch = FilterSketch(1, smooth=0) + sketch.queue = [] if stage == 'initial' else [cp.zeros((2, 1), dtype=cp.int32)] + sketch.max_trees = 1 + sketch.max_nodes = 1 + sketch.lambda_l2 = 1 + probabilities = [] + def select(pi): + probabilities.append(pi.get()) + return cp.asarray([0]) + sketch._select = select + grad = cp.asarray([[1., 2.], [1., 2.]], dtype=cp.float32) * factor + hess = cp.ones_like(grad) * factor + assert bool(cp.isfinite(grad).all() & cp.isfinite(hess).all()) + sketch(grad, hess) + np.testing.assert_allclose(probabilities[0], [.2, .8], rtol=1e-5) + + +@pytest.mark.parametrize('output_weights, expected', [ + ([1e-308, 1e308], np.inf), ([0, 1], -np.log(.9)), +]) +def test_output_weights_preserve_infinite_errors_and_zero_exclusions(gpu, output_weights, expected): + from py_boost.gpu.losses import BCEWithNaNMetricWeighted + with np.errstate(invalid='ignore'): + score = BCEWithNaNMetricWeighted(output_weights)( + [[1., 0.], [1., 0.]], [[0., .1], [0., .1]]) + assert score == pytest.approx(expected) + + +@pytest.mark.parametrize('loss', ['mse', 'msle', 'bce', 'multiclass']) +def test_existing_losses_keep_initialization(gpu, data, loss): + X, _ = data + if loss == 'mse': + y = np.linspace(-1, 1, len(X))[:, None] + elif loss == 'msle': + y = np.linspace(0, 2, len(X))[:, None] + elif loss == 'bce': + y = np.repeat([.2, .8], len(X) // 2)[:, None] + else: + y = (np.arange(len(X)) % 3)[:, None] + weights = np.repeat([1., 9.], len(X) // 2) + model = make_model(loss=loss).fit(X, y, sample_weight=weights) + expected = y.astype(np.float32).mean(axis=0) + if loss == 'bce': + expected = np.log(expected / (1 - expected)) + elif loss == 'msle': + expected = np.log1p(y.astype(np.float32)).mean(axis=0) + elif loss == 'multiclass': + expected = np.zeros(3) + np.testing.assert_allclose(model.base_score, expected, atol=1e-6) + assert np.isfinite(model.predict(X)).all() + + +@pytest.mark.parametrize('sample', [False, True]) +@pytest.mark.parametrize('smooth', [0, .1, 1]) +def test_shared_filter_with_zero_gradient_mse(gpu, data, sample, smooth): + from py_boost.multioutput.sketching import FilterSketch + X, _ = data + y = np.zeros((len(X), 2), dtype=np.float32) + model = make_model(loss='mse', ntrees=2, multioutput_sketch=FilterSketch(1, sample=sample, smooth=smooth), use_hess=True) + model.fit(X, y) + np.testing.assert_array_equal(model.predict(X), y) + + +@pytest.mark.parametrize('loss', ['mse', 'bce']) +def test_shared_early_stopping_reports_nan(gpu, data, loss): + class UndefinedMetric: + alias = 'undefined' + def __call__(self, *args): + return float('nan') + def compare(self, left, right): + return left < right + X, y = data + model = make_model(loss=loss, metric=UndefinedMetric(), es=2) + with pytest.raises(ValueError, match='Early stopping metric.*undefined'): + model.fit(X, y, eval_sets=[{'X': X, 'y': y}]) + + +def test_cross_validation_weighted_initialization(gpu): + from py_boost.cv import CrossValidation + + X = np.random.default_rng(12).normal(size=(96, 6)).astype(np.float32) + y = (X[:, :3] > 0).astype(np.float64) + folds = np.arange(len(X)) % 3 + y[folds == 0, 2] = np.nan + weights = np.linspace(.25, 2, len(X)) + cv = CrossValidation(make_model()) + prediction = cv.fit_predict(X, y, sample_weight=weights, cv=folds) + assert np.isfinite(prediction).all() + for fold, model in enumerate(cv.models): + train = folds != fold + observed = ~np.isnan(y[train]) + eligible_weights = observed * weights[train, None] + expected = (np.where(observed, y[train], 0) * eligible_weights).sum(axis=0) / eligible_weights.sum(axis=0) + np.testing.assert_allclose(model.postprocess_fn(model.base_score), expected, rtol=1e-5) + + +def test_cross_validation_rejects_zero_effective_training_labels(gpu, data): + from py_boost.cv import CrossValidation + + X, y = data + y[32:, 1] = np.nan + y[32:34, 1] = [0, 1] + weights = np.ones(len(y)) + weights[32:34] = 0 + cv = CrossValidation(make_model()) + with pytest.raises(ValueError, match=r'no observed labels.*\[1\]'): + cv.fit_predict(X, y, sample_weight=weights, cv=np.arange(len(y)) // 32) + + +def test_cross_validation_validates_original_label_dtype(gpu, data): + from py_boost.cv import CrossValidation + + X, y = data + y[0, 0] = 1.00000001 + cv = CrossValidation(make_model()) + with pytest.raises(ValueError, match='Observed binary targets must be 0 or 1'): + cv.fit_predict(X, y, cv=np.arange(len(y)) % 2) + + +@pytest.mark.parametrize('weight', [-1., np.nan, np.inf]) +def test_evaluation_weights_are_validated_before_building_trees(gpu, data, monkeypatch, weight): + from py_boost.gpu.tree import DepthwiseTreeBuilder + + def unexpected_tree(*args, **kwargs): + pytest.fail('Invalid evaluation weights must be rejected before building trees') + + monkeypatch.setattr(DepthwiseTreeBuilder, 'build_tree', unexpected_tree) + X, y = data + with pytest.raises(ValueError, match='sample_weight must be finite and nonnegative'): + make_model().fit(X, y, eval_sets=[{'X': X, 'y': y, 'sample_weight': np.full(len(y), weight)}]) + + +@pytest.mark.parametrize('weight', [1e-308, 1e308]) +def test_cross_validation_reports_unrepresentable_training_weights(gpu, data, weight): + from py_boost.cv import CrossValidation + + X, y = data + cv = CrossValidation(make_model()) + with pytest.raises(ValueError, match='Training sample_weight cannot be represented in float32'): + cv.fit_predict(X, y, sample_weight=np.full(len(y), weight), cv=np.arange(len(y)) % 2) + + +@pytest.mark.parametrize('scale', [1e-308, 1e308]) +def test_initial_frequency_rescales_large_and_small_weights(gpu, scale): + from py_boost.gpu.losses import BCEWithNaNLoss + + cp = gpu + loss = BCEWithNaNLoss() + score = loss.base_score(cp.asarray([[0.], [1.]]), np.array([.25, .75]) * scale) + np.testing.assert_allclose(loss.postprocess_output(score).get(), [.75]) + + +def test_initial_frequency_excludes_missing_and_zero_weight_labels(gpu): + from py_boost.gpu.losses import BCEWithNaNLoss + + cp = gpu + y = cp.asarray([[cp.nan, 1], [0, cp.nan], [1, cp.nan], [1, 0]], dtype=cp.float64) + weights = [1e308, 1e-308, 2e-308, 0] + loss = BCEWithNaNLoss() + probabilities = loss.postprocess_output(loss.base_score(y, weights)).get() + np.testing.assert_allclose(probabilities, [2 / 3, 1 - loss.clip_value]) + + +@pytest.mark.parametrize('empty', [False, True]) +def test_filter_zero_hessians_on_cuda(gpu, empty): + from py_boost.multioutput.sketching import FilterSketch + + cp = gpu + grad = cp.asarray([[0., 1.], [0., 2.]], dtype=cp.float32) + hess = cp.asarray([[0., 1.], [0., 1.]], dtype=cp.float32) + if empty: + grad[:] = 0 + hess[:] = 0 + sketch = FilterSketch(1, smooth=0) + sketch.queue = [] + probabilities = [] + def select(pi): + probabilities.append(pi.get()) + return cp.asarray([0]) + sketch._select = select + if empty: + with pytest.raises(ValueError, match='all Hessian sums are zero'): + sketch(grad, hess) + assert probabilities == [] + return + result = sketch(grad, hess) + np.testing.assert_allclose(probabilities[0], [0, 1]) + assert all(bool(cp.isfinite(array).all()) for array in result) + + +def test_training_weight_scale_retains_regularization_effect(gpu): + X = np.linspace(-1, 1, 64, dtype=np.float32)[:, None] + y = (X > 0).astype(np.float32) + ordinary = make_model(max_depth=1, lambda_l2=1).fit(X, y).predict(X) + weighted = make_model(max_depth=1, lambda_l2=1).fit(X, y, sample_weight=np.full(len(X), 5.)).predict(X) + equivalent = make_model(max_depth=1, lambda_l2=.2).fit(X, y).predict(X) + np.testing.assert_allclose(weighted, equivalent, rtol=1e-6, atol=1e-7) + assert not np.allclose(weighted, ordinary) + + +def test_loss_preprocessing_runs_once_per_dataset(gpu, data): + from py_boost.gpu.losses import BCEWithNaNLoss + + class CountingLoss(BCEWithNaNLoss): + calls = 0 + def preprocess_input(self, y_true): + self.calls += 1 + return super().preprocess_input(y_true) + + X, y = data + loss = CountingLoss() + make_model(loss=loss).fit(X, y, eval_sets=[{'X': X, 'y': y}]) + assert loss.calls == 2 + + +def test_legacy_custom_loss_interface(gpu, data): + from py_boost.gpu.losses import BCELoss + + class LegacyLoss: + def __init__(self): + self.impl = BCELoss() + def __getattr__(self, name): + if name in ('prepare_input', 'initial_score'): + raise AttributeError(name) + return getattr(self.impl, name) + def __call__(self, y_true, y_pred): + return self.impl(y_true, y_pred) + + X, y = data + model = make_model(loss=LegacyLoss()).fit(X, y) + assert np.isfinite(model.predict(X)).all() diff --git a/tutorials/Tutorial_4_Handle_null_targets.ipynb b/tutorials/Tutorial_4_Handle_null_targets.ipynb index b4956f7..c1bc336 100644 --- a/tutorials/Tutorial_4_Handle_null_targets.ipynb +++ b/tutorials/Tutorial_4_Handle_null_targets.ipynb @@ -66,9 +66,15 @@ "source": [ "### Use the built-in loss and metrics\n", "\n", - "Every training target column must contain at least one observed label. A fully unknown training column raises `ValueError` listing its zero-based index; it is not silently removed or assigned a probability of 0.5. Both training and evaluation targets are checked for values other than `0`, `1`, and `NaN` before training.\n", + "Each training target column must contain an observed label with positive row weight. Without `sample_weight`, every row has weight 1. Both fully unknown columns and columns whose observed labels all have zero weight raise `ValueError` with zero-based column indices before initialization or tree construction. The same check applies to the training partition of each cross-validation fold. No output is dropped or assigned a probability of 0.5.\n", "\n", - "Use positive `lambda_l2` so leaves without observed labels have finite updates. The loss also accepts the string aliases `bce_nan` and `multilabel_nan`; its default metric is `NaNBCEMetric`.\n" + "Initial probabilities use weighted frequencies of observed positive labels; zero-weight rows do not affect them. Training and evaluation labels are validated as `0`, `1`, or `NaN` in their original dtype before float32 conversion, including labels on zero-weight rows. For example, float64 label `1.00000001` is rejected.\n", + "\n", + "Row weights must be finite and nonnegative. Training uses float32 weights and raises a representation error if a finite weight overflows or a positive weight becomes zero. Training weights are not normalized, preserving their effect on regularization. Evaluation weights stay float64 through `fit`, allowing metrics to rescale weights such as `1e308` and `1e-308` safely. These strict validation and initialization rules apply only to `BCEWithNaNLoss`; existing losses retain their behavior.\n", + "\n", + "Use positive `lambda_l2` so leaves without observed labels have finite updates. The loss also accepts the string aliases `bce_nan` and `multilabel_nan`; its default metric is `NaNBCEMetric`.\n", + "\n", + "`FilterSketch` excludes outputs with zero Hessian mass and falls back to uniform selection over eligible outputs when their importances are all zero. If no output has positive Hessian mass, it raises an explicit error. Importance squaring and accumulation use float64. Zero gradients at an optimum are valid and are separate from missing training labels.\n" ] }, { From 55c7c302b236c25cbffd85aa74029f9cef7675cd Mon Sep 17 00:00:00 2001 From: Rinchin Date: Fri, 30 Oct 2026 15:18:16 +0300 Subject: [PATCH 4/5] Restore float32 FilterSketch scratch calculations --- README.md | 12 +++++++++--- py_boost/multioutput/sketching.py | 6 +++--- tests/test_nan_training_contract.py | 4 ++-- tutorials/Tutorial_4_Handle_null_targets.ipynb | 2 +- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index bc501db..9728c42 100644 --- a/README.md +++ b/README.md @@ -145,9 +145,15 @@ Outputs with zero total Hessian are excluded from selection. If all remaining output importances are zero, selection uses a uniform distribution over the eligible outputs; if none remain, it raises an error identifying the lack of positive Hessian mass. The separate training-column check requires an observed -label with positive row weight in each training output. Importance squaring and -accumulation use float64 to avoid float32 overflow and underflow. Zero gradients -at an optimum are valid and do not trigger a missing-training-label error. +label with positive row weight in each training output. Gradient squaring and +per-leaf accumulation use float32 to limit GPU memory; reductions and probability +normalization use float64. With the default BCE clipping and ordinary row +weights, gradient magnitudes are well within the float32 range. Extreme weights +or gradient scales can still cause overflow or underflow during squaring, and +roundoff can affect importance estimates when leaf gradients nearly cancel. +Float64 reductions do not remove these float32 training limitations. Zero +gradients at an optimum are valid and do not trigger a missing-training-label +error. For `BCEWithNaNLoss`, row weights are validated as finite and nonnegative before conversion. Training weights are stored as float32: overflow to infinity diff --git a/py_boost/multioutput/sketching.py b/py_boost/multioutput/sketching.py index 25b889a..3b1c291 100644 --- a/py_boost/multioutput/sketching.py +++ b/py_boost/multioutput/sketching.py @@ -252,14 +252,14 @@ def _calc_weights(self, grad, hess): return cp.ones(grad.shape[1], dtype=cp.float64) if len(self.queue) == 0: - loss = cp.square(grad, dtype=cp.float64).sum(axis=0) + loss = (grad ** 2).sum(axis=0, dtype=cp.float64) denominator = hess.sum(axis=0, dtype=cp.float64) return cp.where(denominator > 0, loss / cp.where(denominator > 0, denominator, 1), 0) grad_sum = cp.zeros((len(self.queue), self.max_trees, self.max_nodes, - grad.shape[1],), dtype=cp.float64) + grad.shape[1],), dtype=cp.float32) hess_sum = cp.zeros((len(self.queue), self.max_trees, self.max_nodes, - hess.shape[1],), dtype=cp.float64) + hess.shape[1],), dtype=cp.float32) for n, prev_iter in enumerate(self.queue): for i in range(prev_iter.shape[1]): diff --git a/tests/test_nan_training_contract.py b/tests/test_nan_training_contract.py index d731208..ff8eb2d 100644 --- a/tests/test_nan_training_contract.py +++ b/tests/test_nan_training_contract.py @@ -83,8 +83,8 @@ def test_extreme_evaluation_weights_through_fit(gpu, data, weight, metric): @pytest.mark.parametrize('stage', ['initial', 'queued']) -@pytest.mark.parametrize('factor', [1., 1e-30, 1e20]) -def test_filter_importance_precision(gpu, stage, factor): +@pytest.mark.parametrize('factor', [1e-3, 1., 1e3]) +def test_filter_importance_with_moderate_scales(gpu, stage, factor): from py_boost.multioutput.sketching import FilterSketch cp = gpu sketch = FilterSketch(1, smooth=0) diff --git a/tutorials/Tutorial_4_Handle_null_targets.ipynb b/tutorials/Tutorial_4_Handle_null_targets.ipynb index c1bc336..74e1e5a 100644 --- a/tutorials/Tutorial_4_Handle_null_targets.ipynb +++ b/tutorials/Tutorial_4_Handle_null_targets.ipynb @@ -74,7 +74,7 @@ "\n", "Use positive `lambda_l2` so leaves without observed labels have finite updates. The loss also accepts the string aliases `bce_nan` and `multilabel_nan`; its default metric is `NaNBCEMetric`.\n", "\n", - "`FilterSketch` excludes outputs with zero Hessian mass and falls back to uniform selection over eligible outputs when their importances are all zero. If no output has positive Hessian mass, it raises an explicit error. Importance squaring and accumulation use float64. Zero gradients at an optimum are valid and are separate from missing training labels.\n" + "`FilterSketch` excludes outputs with zero Hessian mass and falls back to uniform selection over eligible outputs when their importances are all zero. If no output has positive Hessian mass, it raises an explicit error. Gradient squaring and per-leaf accumulation use float32 to limit GPU memory; reductions and probability normalization use float64. Extreme weights or gradient scales can still overflow or underflow during squaring, and roundoff can affect nearly cancelling leaf gradients. Zero gradients at an optimum are valid and are separate from missing training labels.\n" ] }, { From a74f2208144effcf1466c266e57ddee658b427a6 Mon Sep 17 00:00:00 2001 From: Rinchin Date: Fri, 30 Oct 2026 15:26:06 +0300 Subject: [PATCH 5/5] Execute Tutorial 4 and refresh notebook outputs --- .../Tutorial_4_Handle_null_targets.ipynb | 118 ++++++++++++++++-- 1 file changed, 105 insertions(+), 13 deletions(-) diff --git a/tutorials/Tutorial_4_Handle_null_targets.ipynb b/tutorials/Tutorial_4_Handle_null_targets.ipynb index 74e1e5a..718f65c 100644 --- a/tutorials/Tutorial_4_Handle_null_targets.ipynb +++ b/tutorials/Tutorial_4_Handle_null_targets.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -46,9 +46,15 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": "CPU times: user 2.63 s, sys: 146 ms, total: 2.78 s\nWall time: 524 ms\n" + } + ], "source": [ "%%time\n", "X, y = make_regression(150000, 100, n_targets=10, random_state=42)\n", @@ -79,7 +85,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -103,7 +109,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -113,9 +119,88 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": "[15:24:58] Stdout logging level is INFO.\n" + }, + { + "output_type": "stream", + "name": "stdout", + "text": "[15:24:58] GDBT train starts. Max iter 1000, early stopping rounds 200\n" + }, + { + "output_type": "stream", + "name": "stdout", + "text": "[15:24:58] Iter 0; Sample 0, AUC_nan = 0.7979585947464197; \n" + }, + { + "output_type": "stream", + "name": "stdout", + "text": "[15:25:00] Iter 100; Sample 0, AUC_nan = 0.9692103969285927; \n" + }, + { + "output_type": "stream", + "name": "stdout", + "text": "[15:25:02] Iter 200; Sample 0, AUC_nan = 0.9789776294844492; \n" + }, + { + "output_type": "stream", + "name": "stdout", + "text": "[15:25:04] Iter 300; Sample 0, AUC_nan = 0.9847043948819084; \n" + }, + { + "output_type": "stream", + "name": "stdout", + "text": "[15:25:07] Iter 400; Sample 0, AUC_nan = 0.988497967529567; \n" + }, + { + "output_type": "stream", + "name": "stdout", + "text": "[15:25:09] Iter 500; Sample 0, AUC_nan = 0.9908048438238338; \n" + }, + { + "output_type": "stream", + "name": "stdout", + "text": "[15:25:11] Iter 600; Sample 0, AUC_nan = 0.9923416720282536; \n" + }, + { + "output_type": "stream", + "name": "stdout", + "text": "[15:25:13] Iter 700; Sample 0, AUC_nan = 0.9935401806228439; \n" + }, + { + "output_type": "stream", + "name": "stdout", + "text": "[15:25:15] Iter 800; Sample 0, AUC_nan = 0.9942690161033253; \n" + }, + { + "output_type": "stream", + "name": "stdout", + "text": "[15:25:17] Iter 900; Sample 0, AUC_nan = 0.9948963169542316; \n" + }, + { + "output_type": "stream", + "name": "stdout", + "text": "[15:25:19] Iter 999; Sample 0, AUC_nan = 0.9954203098020314; \n" + }, + { + "output_type": "stream", + "name": "stdout", + "text": "CPU times: user 26.8 s, sys: 309 ms, total: 27.1 s\nWall time: 25.4 s\n" + }, + { + "output_type": "execute_result", + "metadata": {}, + "data": { + "text/plain": "" + }, + "execution_count": 5 + } + ], "source": [ "%%time\n", "model = GradientBoosting(\n", @@ -138,8 +223,14 @@ { "cell_type": "code", "metadata": {}, - "execution_count": null, - "outputs": [], + "execution_count": 6, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": "[0.99491351 0.99557796 0.99559866 0.99480372 0.9953295 0.99574239\n 0.99586409 0.99561081 0.99534213 nan]\n" + } + ], "source": [ "prediction = model.predict(X_test)\n", "validation_auc = NaNAucMetricByClass()(y_test, prediction)\n", @@ -157,15 +248,16 @@ "name": "python3" }, "language_info": { + "name": "python", + "version": "3.12.3", + "mimetype": "text/x-python", "codemirror_mode": { "name": "ipython", "version": 3 }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", + "pygments_lexer": "ipython3", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" + "file_extension": ".py" } }, "nbformat": 4,