diff --git a/tests/unit_tests/model_validation/sklearn/test_CalibrationCurve.py b/tests/unit_tests/model_validation/sklearn/test_CalibrationCurve.py index cd5b33bc0..a733c1f03 100644 --- a/tests/unit_tests/model_validation/sklearn/test_CalibrationCurve.py +++ b/tests/unit_tests/model_validation/sklearn/test_CalibrationCurve.py @@ -2,6 +2,7 @@ import numpy as np import pandas as pd +from sklearn.calibration import calibration_curve from sklearn.linear_model import LogisticRegression import validmind as vm @@ -38,6 +39,36 @@ def test_binary_runs(self): result = CalibrationCurve(model=model, dataset=ds) self.assertIsNotNone(result) + def test_binary_non_standard_encoding_runs(self): + # Follow-up to the #534 review: a two-class target encoded as {0, 4} passes + # the multiclass guard but previously hit sklearn's "pos_label is not + # specified" error. Resolving the positive label to the largest value fixes + # it. + ds, model = _dataset_and_model([0, 4], seed=3) + result = CalibrationCurve(model=model, dataset=ds) + self.assertIsNotNone(result) + + def test_binary_string_labels_run(self): + # String labels ("bad"/"good") are the same failure family and must not + # raise. + ds, model = _dataset_and_model(["bad", "good"], seed=4) + result = CalibrationCurve(model=model, dataset=ds) + self.assertIsNotNone(result) + + def test_standard_binary_curve_unchanged(self): + # Backward compatibility: for {0, 1} the raw curve must be byte-identical to + # sklearn's default (which resolves pos_label to 1) so existing results are + # unaffected. + ds, model = _dataset_and_model([0, 1], seed=2) + _, raw_data = CalibrationCurve(model=model, dataset=ds) + expected_true, expected_pred = calibration_curve( + ds.y, ds.y_prob(model), n_bins=10 + ) + np.testing.assert_array_equal(raw_data.observed_frequency, expected_true) + np.testing.assert_array_equal( + raw_data.mean_predicted_probability, expected_pred + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit_tests/model_validation/sklearn/test_WeakspotsDiagnosis.py b/tests/unit_tests/model_validation/sklearn/test_WeakspotsDiagnosis.py index 92be8d2f4..2c07bf2c4 100644 --- a/tests/unit_tests/model_validation/sklearn/test_WeakspotsDiagnosis.py +++ b/tests/unit_tests/model_validation/sklearn/test_WeakspotsDiagnosis.py @@ -1,3 +1,4 @@ +import functools import unittest import numpy as np @@ -222,5 +223,77 @@ def test_standard_binary_matches_default_precision(self): self.assertGreater(checked, 0) +class TestWeakspotsDiagnosisCustomMetrics(unittest.TestCase): + """Follow-up to the #534 review: user-supplied custom metrics must keep their + own averaging. ``functools.partial(f1_score, average="weighted")`` still exposes + ``average`` via ``inspect.signature``, so the old unconditional ``_apply_averaging`` + silently rebound it to the resolved default (macro/binary).""" + + @staticmethod + def _first_nonempty_slice(dataset, model, feature): + """Return (region, y_true, y_pred) for the first non-empty bin of a feature, + mirroring WeakspotsDiagnosis' own binning so recomputation lines up.""" + target = dataset.target_column + pred_col = dataset.prediction_column(model) + binned = dataset._df.copy() + binned["bin"] = pd.cut(binned[feature], bins=10) + for region, slice_df in binned.groupby("bin", observed=True): + if slice_df.empty: + continue + y_true = slice_df[target].values + y_pred = slice_df[pred_col].astype(slice_df[target].dtype).values + return region, y_true, y_pred + return None, None, None + + def test_custom_metric_keeps_user_averaging(self): + # Multiclass target: the resolved default is macro averaging, so the old code + # rebound the user's weighted partial to macro. The results table must carry + # the user's *weighted* F1, not macro. + train, test, model = _train_test_pair("custom_weighted", [0, 1, 2]) + weighted_f1 = functools.partial(skm.f1_score, average="weighted") + result = WeakspotsDiagnosis( + datasets=[train, test], + model=model, + metrics={"f1": weighted_f1}, + thresholds={"f1": 0.5}, + ) + df = result[0] + + feature = train.feature_columns[0] + region, y_true, y_pred = self._first_nonempty_slice(train, model, feature) + self.assertIsNotNone(region) + expected_weighted = weighted_f1(y_true, y_pred) + expected_macro = skm.f1_score(y_true, y_pred, average="macro") + # The scenario is only meaningful when the two averagings disagree. + self.assertNotAlmostEqual(expected_weighted, expected_macro) + + row = df[ + (df["Feature"] == feature) + & (df["Slice"] == str(region)) + & (df["Dataset"] == train.input_id) + ] + self.assertAlmostEqual(row["F1"].iloc[0], expected_weighted) + + def test_default_metrics_still_get_macro_rebinding(self): + # The defaults path must keep resolving/binding averaging: on a multiclass + # target the default F1 must equal macro (sklearn's binary default would + # raise), confirming the fix only skips rebinding for custom metrics. + train, test, model = _train_test_pair("default_macro", [0, 1, 2]) + result = WeakspotsDiagnosis(datasets=[train, test], model=model) + df = result[0] + + feature = train.feature_columns[0] + region, y_true, y_pred = self._first_nonempty_slice(train, model, feature) + self.assertIsNotNone(region) + expected_macro = skm.f1_score(y_true, y_pred, average="macro", zero_division=0) + + row = df[ + (df["Feature"] == feature) + & (df["Slice"] == str(region)) + & (df["Dataset"] == train.input_id) + ] + self.assertAlmostEqual(row["F1"].iloc[0], expected_macro) + + if __name__ == "__main__": unittest.main() diff --git a/validmind/tests/model_validation/sklearn/CalibrationCurve.py b/validmind/tests/model_validation/sklearn/CalibrationCurve.py index b3075c2fc..475e2046e 100644 --- a/validmind/tests/model_validation/sklearn/CalibrationCurve.py +++ b/validmind/tests/model_validation/sklearn/CalibrationCurve.py @@ -2,6 +2,7 @@ # Refer to the LICENSE file in the root of this repository for details. # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial +import inspect from typing import Tuple import numpy as np @@ -80,8 +81,19 @@ def CalibrationCurve( "Calibration Curve is only supported for binary classification models" ) + # A two-class target encoded outside {0, 1} (e.g. {0, 4} or string labels) + # passes the guard above but hits sklearn's "pos_label is not specified" error. + # Pass the largest label as the positive class, matching the "largest label is + # positive" convention in _diagnosis_metrics.resolve_averaging; for {0, 1} this + # is byte-identical to sklearn's default. pos_label was added to + # calibration_curve in scikit-learn 1.1, so guard by signature inspection to + # keep pre-1.1 installs working (pyproject sets no scikit-learn floor). + kwargs = {} + if "pos_label" in inspect.signature(calibration_curve).parameters: + kwargs["pos_label"] = np.unique(dataset.y).max() + prob_true, prob_pred = calibration_curve( - dataset.y, dataset.y_prob(model), n_bins=n_bins + dataset.y, dataset.y_prob(model), n_bins=n_bins, **kwargs ) # Create DataFrame for raw data diff --git a/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py b/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py index bef73c9fe..9e857db1b 100644 --- a/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py +++ b/validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py @@ -263,6 +263,11 @@ def WeakspotsDiagnosis( "Column(s) provided in features_columns do not exist in the dataset" ) + # Custom callables own their kwargs (e.g. partial(f1_score, average="weighted")), + # so averaging is only bound onto the defaults; rebinding user metrics would + # silently override their explicit averaging/pos_label choices. + using_default_metrics = metrics is None + metrics, plot_thresholds, pass_thresholds = _prepare_metrics_and_thresholds( metrics, thresholds ) @@ -270,8 +275,9 @@ def WeakspotsDiagnosis( # Bind averaging options so label-based metrics work for multiclass targets and # for binary targets encoded outside {0, 1} (e.g. {0, 4}) instead of raising on # scikit-learn's default average="binary"/pos_label=1. - average, pos_label = _resolve_averaging(datasets, model) - metrics = _apply_averaging(metrics, average, pos_label) + if using_default_metrics: + average, pos_label = _resolve_averaging(datasets, model) + metrics = _apply_averaging(metrics, average, pos_label) results_headers = ["Slice", "Number of Records", "Feature"] results_headers.extend(metrics.keys())