fix: CalibrationCurve non-{0,1} binary crash and Weakspots custom-metric override

tests/unit_tests/model_validation/sklearn/test_CalibrationCurve.py

@@ -2,6 +2,7 @@
 
 import numpy as np
 import pandas as pd
+from sklearn.calibration import calibration_curve
 from sklearn.linear_model import LogisticRegression
 
 import validmind as vm
@@ -38,6 +39,36 @@ def test_binary_runs(self):
         result = CalibrationCurve(model=model, dataset=ds)
         self.assertIsNotNone(result)
 
+    def test_binary_non_standard_encoding_runs(self):
+        # Follow-up to the #534 review: a two-class target encoded as {0, 4} passes
+        # the multiclass guard but previously hit sklearn's "pos_label is not
+        # specified" error. Resolving the positive label to the largest value fixes
+        # it.
+        ds, model = _dataset_and_model([0, 4], seed=3)
+        result = CalibrationCurve(model=model, dataset=ds)
+        self.assertIsNotNone(result)
+
+    def test_binary_string_labels_run(self):
+        # String labels ("bad"/"good") are the same failure family and must not
+        # raise.
+        ds, model = _dataset_and_model(["bad", "good"], seed=4)
+        result = CalibrationCurve(model=model, dataset=ds)
+        self.assertIsNotNone(result)
+
+    def test_standard_binary_curve_unchanged(self):
+        # Backward compatibility: for {0, 1} the raw curve must be byte-identical to
+        # sklearn's default (which resolves pos_label to 1) so existing results are
+        # unaffected.
+        ds, model = _dataset_and_model([0, 1], seed=2)
+        _, raw_data = CalibrationCurve(model=model, dataset=ds)
+        expected_true, expected_pred = calibration_curve(
+            ds.y, ds.y_prob(model), n_bins=10
+        )
+        np.testing.assert_array_equal(raw_data.observed_frequency, expected_true)
+        np.testing.assert_array_equal(
+            raw_data.mean_predicted_probability, expected_pred
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

tests/unit_tests/model_validation/sklearn/test_WeakspotsDiagnosis.py

@@ -1,3 +1,4 @@
+import functools
 import unittest
 
 import numpy as np
@@ -222,5 +223,77 @@ def test_standard_binary_matches_default_precision(self):
         self.assertGreater(checked, 0)
 
 
+class TestWeakspotsDiagnosisCustomMetrics(unittest.TestCase):
+    """Follow-up to the #534 review: user-supplied custom metrics must keep their
+    own averaging. ``functools.partial(f1_score, average="weighted")`` still exposes
+    ``average`` via ``inspect.signature``, so the old unconditional ``_apply_averaging``
+    silently rebound it to the resolved default (macro/binary)."""
+
+    @staticmethod
+    def _first_nonempty_slice(dataset, model, feature):
+        """Return (region, y_true, y_pred) for the first non-empty bin of a feature,
+        mirroring WeakspotsDiagnosis' own binning so recomputation lines up."""
+        target = dataset.target_column
+        pred_col = dataset.prediction_column(model)
+        binned = dataset._df.copy()
+        binned["bin"] = pd.cut(binned[feature], bins=10)
+        for region, slice_df in binned.groupby("bin", observed=True):
+            if slice_df.empty:
+                continue
+            y_true = slice_df[target].values
+            y_pred = slice_df[pred_col].astype(slice_df[target].dtype).values
+            return region, y_true, y_pred
+        return None, None, None
+
+    def test_custom_metric_keeps_user_averaging(self):
+        # Multiclass target: the resolved default is macro averaging, so the old code
+        # rebound the user's weighted partial to macro. The results table must carry
+        # the user's *weighted* F1, not macro.
+        train, test, model = _train_test_pair("custom_weighted", [0, 1, 2])
+        weighted_f1 = functools.partial(skm.f1_score, average="weighted")
+        result = WeakspotsDiagnosis(
+            datasets=[train, test],
+            model=model,
+            metrics={"f1": weighted_f1},
+            thresholds={"f1": 0.5},
+        )
+        df = result[0]
+
+        feature = train.feature_columns[0]
+        region, y_true, y_pred = self._first_nonempty_slice(train, model, feature)
+        self.assertIsNotNone(region)
+        expected_weighted = weighted_f1(y_true, y_pred)
+        expected_macro = skm.f1_score(y_true, y_pred, average="macro")
+        # The scenario is only meaningful when the two averagings disagree.
+        self.assertNotAlmostEqual(expected_weighted, expected_macro)
+
+        row = df[
+            (df["Feature"] == feature)
+            & (df["Slice"] == str(region))
+            & (df["Dataset"] == train.input_id)
+        ]
+        self.assertAlmostEqual(row["F1"].iloc[0], expected_weighted)
+
+    def test_default_metrics_still_get_macro_rebinding(self):
+        # The defaults path must keep resolving/binding averaging: on a multiclass
+        # target the default F1 must equal macro (sklearn's binary default would
+        # raise), confirming the fix only skips rebinding for custom metrics.
+        train, test, model = _train_test_pair("default_macro", [0, 1, 2])
+        result = WeakspotsDiagnosis(datasets=[train, test], model=model)
+        df = result[0]
+
+        feature = train.feature_columns[0]
+        region, y_true, y_pred = self._first_nonempty_slice(train, model, feature)
+        self.assertIsNotNone(region)
+        expected_macro = skm.f1_score(y_true, y_pred, average="macro", zero_division=0)
+
+        row = df[
+            (df["Feature"] == feature)
+            & (df["Slice"] == str(region))
+            & (df["Dataset"] == train.input_id)
+        ]
+        self.assertAlmostEqual(row["F1"].iloc[0], expected_macro)
+
+
 if __name__ == "__main__":
     unittest.main()

validmind/tests/model_validation/sklearn/CalibrationCurve.py

@@ -2,6 +2,7 @@
 # Refer to the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import inspect
 from typing import Tuple
 
 import numpy as np
@@ -80,8 +81,19 @@ def CalibrationCurve(
             "Calibration Curve is only supported for binary classification models"
         )
 
+    # A two-class target encoded outside {0, 1} (e.g. {0, 4} or string labels)
+    # passes the guard above but hits sklearn's "pos_label is not specified" error.
+    # Pass the largest label as the positive class, matching the "largest label is
+    # positive" convention in _diagnosis_metrics.resolve_averaging; for {0, 1} this
+    # is byte-identical to sklearn's default. pos_label was added to
+    # calibration_curve in scikit-learn 1.1, so guard by signature inspection to
+    # keep pre-1.1 installs working (pyproject sets no scikit-learn floor).
+    kwargs = {}
+    if "pos_label" in inspect.signature(calibration_curve).parameters:
+        kwargs["pos_label"] = np.unique(dataset.y).max()
+
     prob_true, prob_pred = calibration_curve(
-        dataset.y, dataset.y_prob(model), n_bins=n_bins
+        dataset.y, dataset.y_prob(model), n_bins=n_bins, **kwargs
     )
 
     # Create DataFrame for raw data

validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py

@@ -263,15 +263,21 @@ def WeakspotsDiagnosis(
             "Column(s) provided in features_columns do not exist in the dataset"
         )
 
+    # Custom callables own their kwargs (e.g. partial(f1_score, average="weighted")),
+    # so averaging is only bound onto the defaults; rebinding user metrics would
+    # silently override their explicit averaging/pos_label choices.
+    using_default_metrics = metrics is None
+
     metrics, plot_thresholds, pass_thresholds = _prepare_metrics_and_thresholds(
         metrics, thresholds
     )
 
     # Bind averaging options so label-based metrics work for multiclass targets and
     # for binary targets encoded outside {0, 1} (e.g. {0, 4}) instead of raising on
     # scikit-learn's default average="binary"/pos_label=1.
-    average, pos_label = _resolve_averaging(datasets, model)
-    metrics = _apply_averaging(metrics, average, pos_label)
+    if using_default_metrics:
+        average, pos_label = _resolve_averaging(datasets, model)
+        metrics = _apply_averaging(metrics, average, pos_label)
 
     results_headers = ["Slice", "Number of Records", "Feature"]
     results_headers.extend(metrics.keys())