From 727059574d09ceab5c5e9f1e60e96ceb122b5ffd Mon Sep 17 00:00:00 2001 From: lenarddome Date: Wed, 11 Nov 2026 10:41:45 +0100 Subject: [PATCH 1/4] fix(metad): infer responses from accuracy in count_trials (#106) Co-Authored-By: Claude Opus 5.5 --- cpm/utils/metad.py | 82 +++++++++++++++++++++++++++-------- test/utils/test_metad.py | 94 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+), 18 deletions(-) create mode 100644 test/utils/test_metad.py diff --git a/cpm/utils/metad.py b/cpm/utils/metad.py index a99bafca..e74d4309 100644 --- a/cpm/utils/metad.py +++ b/cpm/utils/metad.py @@ -1,8 +1,14 @@ import numpy as np import pandas as pd + +def _is_column(data: pd.DataFrame, name) -> bool: + """Whether `name` is given and names a column of `data`.""" + return name is not None and name in data.columns + + def count_trials( - data=pd.DataFrame, + data: pd.DataFrame | None = None, stimuli: str = "Stimuli", responses: str = "Responses", accuracy: str = "Accuracy", @@ -110,25 +116,65 @@ def count_trials( """ if isinstance(data, pd.DataFrame): - stimuli = data[stimuli].to_numpy() - confidence = data[confidence].to_numpy() - if accuracy in data: - accuracy = data[accuracy].to_numpy() - if responses in data: - responses = data[responses].to_numpy() + stimuli = data[stimuli] + confidence = data[confidence] + responses = data[responses] if _is_column(data, responses) else None + accuracy = data[accuracy] if _is_column(data, accuracy) else None elif data is not None: raise ValueError("`Data` should be a DataFrame") - - # Check data consistency - tempstim, tempresp, tempratg = [], [], [] - for s, rp, rt in zip(stimuli, responses, confidence): - if (s == 0 or s == 1) and (rp == 0 or rp == 1) and (rt >= 1 and rt <= nRatings): - tempstim.append(s) - tempresp.append(rp) - tempratg.append(rt) - stimuli = tempstim - responses = tempresp - confidence = tempratg + else: + # In array mode, a string is the column-name default left in place. + responses = None if isinstance(responses, str) else responses + accuracy = None if isinstance(accuracy, str) else accuracy + + if responses is None and accuracy is None: + raise ValueError( + "count_trials needs `responses`, `accuracy`, or both; neither was provided." + ) + + stimuli = np.asarray(stimuli, dtype=float) + confidence = np.asarray(confidence, dtype=float) + inputs = [stimuli, confidence] + if responses is not None: + responses = np.asarray(responses, dtype=float) + inputs.append(responses) + if accuracy is not None: + accuracy = np.asarray(accuracy, dtype=float) + inputs.append(accuracy) + if len({len(x) for x in inputs}) > 1: + raise ValueError( + "`stimuli`, `responses`, `accuracy` and `confidence` must have the same " + f"length; got lengths {[len(x) for x in inputs]}." + ) + + binary_stimuli = np.isin(stimuli, (0, 1)) + if accuracy is not None: + binary_accuracy = np.isin(accuracy, (0, 1)) + if responses is None: + # A correct response matches the stimulus; an incorrect one does not. + # Trials with invalid stimuli or accuracy get NaN and are dropped below. + responses = np.where( + binary_stimuli & binary_accuracy, + np.where(accuracy == 1, stimuli, 1 - stimuli), + np.nan, + ) + + valid = ( + binary_stimuli + & np.isin(responses, (0, 1)) + & (confidence >= 1) + & (confidence <= nRatings) + ) + if accuracy is not None: + valid &= binary_accuracy + if not valid.any(): + raise ValueError( + "count_trials found no valid trials: `stimuli` and responses must be 0 " + f"or 1, and `confidence` must be in [1, {nRatings}]." + ) + stimuli = stimuli[valid] + responses = responses[valid] + confidence = confidence[valid] if padAmount is None: padAmount = 1 / (2 * nRatings) diff --git a/test/utils/test_metad.py b/test/utils/test_metad.py new file mode 100644 index 00000000..09c2d4f2 --- /dev/null +++ b/test/utils/test_metad.py @@ -0,0 +1,94 @@ +import numpy as np +import pandas as pd +import pytest + +from cpm.utils.metad import count_trials + +STIM = [0, 1, 0, 0, 1, 1, 1, 1] +ACC = [0, 1, 1, 1, 0, 0, 1, 1] +CONF = [1, 2, 3, 4, 4, 3, 2, 1] +RESP = [1, 1, 0, 0, 0, 0, 1, 1] # inferred from STIM and ACC +EXPECTED_S1 = [1, 1, 0, 0, 1, 0, 0, 0] +EXPECTED_S2 = [1, 1, 0, 0, 1, 2, 0, 0] + + +def assert_expected(result): + nR_S1, nR_S2 = result + np.testing.assert_array_equal(nR_S1, EXPECTED_S1) + np.testing.assert_array_equal(nR_S2, EXPECTED_S2) + + +def test_responses_only_dataframe(): + data = pd.DataFrame({"Stimuli": STIM, "Responses": RESP, "Confidence": CONF}) + assert_expected(count_trials(data, nRatings=4)) + + +def test_infers_responses_from_accuracy_dataframe(): + data = pd.DataFrame({"Stimuli": STIM, "Accuracy": ACC, "Confidence": CONF}) + assert_expected(count_trials(data, nRatings=4)) + + +def test_infers_responses_from_accuracy_arrays(): + assert_expected( + count_trials(stimuli=STIM, accuracy=ACC, confidence=CONF, nRatings=4) + ) + + +def test_responses_arrays(): + assert_expected( + count_trials( + stimuli=STIM, responses=RESP, accuracy=None, confidence=CONF, nRatings=4 + ) + ) + + +def test_neither_responses_nor_accuracy_raises(): + data = pd.DataFrame({"Stimuli": STIM, "Confidence": CONF}) + with pytest.raises(ValueError, match="responses.*accuracy"): + count_trials(data, nRatings=4) + + +def test_invalid_accuracy_trials_dropped(): + data = pd.DataFrame( + {"Stimuli": STIM + [0], "Accuracy": ACC + [2], "Confidence": CONF + [1]} + ) + assert_expected(count_trials(data, nRatings=4)) + + +def test_nan_accuracy_trials_dropped(): + data = pd.DataFrame( + {"Stimuli": STIM + [1], "Accuracy": ACC + [np.nan], "Confidence": CONF + [2]} + ) + assert_expected(count_trials(data, nRatings=4)) + + +def test_boolean_accuracy(): + data = pd.DataFrame( + {"Stimuli": STIM, "Accuracy": [bool(a) for a in ACC], "Confidence": CONF} + ) + assert_expected(count_trials(data, nRatings=4)) + + +def test_no_valid_trials_raises(): + data = pd.DataFrame( + {"Stimuli": [s + 2 for s in STIM], "Accuracy": ACC, "Confidence": CONF} + ) + with pytest.raises(ValueError, match="no valid trials"): + count_trials(data, nRatings=4) + + +def test_mismatched_lengths_raise(): + with pytest.raises(ValueError, match="length"): + count_trials(stimuli=STIM, accuracy=ACC[:-1], confidence=CONF, nRatings=4) + + +def test_padding_default_amount(): + data = pd.DataFrame({"Stimuli": STIM, "Responses": RESP, "Confidence": CONF}) + nR_S1, nR_S2 = count_trials(data, nRatings=4, padding=True) + np.testing.assert_allclose(nR_S1, np.array(EXPECTED_S1) + 1 / 8) + np.testing.assert_allclose(nR_S2, np.array(EXPECTED_S2) + 1 / 8) + + +def test_non_dataframe_data_raises(): + with pytest.raises(ValueError, match="DataFrame"): + count_trials(data=[1, 2, 3]) From 1f6da153b3146da52dcff5ef303e7e9b337c44de Mon Sep 17 00:00:00 2001 From: lenarddome Date: Wed, 11 Nov 2026 10:47:44 +0100 Subject: [PATCH 2/4] fix(metad): check responses against accuracy in count_trials (#106) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++ cpm/utils/metad.py | 68 +++++++++++++++------- test/applications/test_signal_detection.py | 2 +- test/utils/test_metad.py | 31 ++++++++++ 4 files changed, 84 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2be0ab30..c603e279 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed + +- `cpm.utils.metad.count_trials` now infers responses from accuracy when only accuracy is given, checks that responses and accuracy agree when both are given, and accepts arrays without a DataFrame; it previously returned all-zero counts when the responses column was missing ([#106](https://github.com/DevComPsy/cpm/issues/106)) + ## [0.26.0] - 2026-10-16 ### Added diff --git a/cpm/utils/metad.py b/cpm/utils/metad.py index e74d4309..25900047 100644 --- a/cpm/utils/metad.py +++ b/cpm/utils/metad.py @@ -27,18 +27,24 @@ def count_trials( Parameters ---------- data : - Dataframe containing stimuli, accuracy and confidence ratings. + DataFrame containing the columns below, or `None` to pass the stimuli, + responses, accuracy and confidence ratings as arrays. Default is + `None`. stimuli : Stimuli ID (0 or 1). If a dataframe is provided, should be the name of the column containing the stimuli ID. Default is `'Stimuli'`. responses : - Response (0 or 1). If a dataframe is provided, should be the - name of the column containing the response accuracy. Default is - `'Responses'`. + Response (0 or 1). Optional, but at least one of `responses` and + `accuracy` is required. If a dataframe is provided, should be the name + of the column containing the responses; a name that is not a column of + `data` counts as not provided. With arrays, `None` or a string counts + as not provided. Default is `'Responses'`. accuracy : - Response accuracy (0 or 1). If a dataframe is provided, should be the - name of the column containing the response accuracy. Default is - `'Accuracy'`. + Response accuracy (0 or 1). Optional, but at least one of `responses` + and `accuracy` is required. If a dataframe is provided, should be the + name of the column containing the response accuracy; a name that is + not a column of `data` counts as not provided. With arrays, `None` or + a string counts as not provided. Default is `'Accuracy'`. confidence : Confidence ratings. If a dataframe is provided, should be the name of the column containing the confidence ratings. Default is @@ -51,10 +57,9 @@ def count_trials( If `True`, each response count in the output has the value of padAmount added to it. Padding cells is desirable if trial counts of 0 interfere with model fitting. If False, trial counts are not manipulated and 0s - may be present in the response count output. Default value for padding - is 0. + may be present in the response count output. Default is `False`. padAmount : - The value to add to each response count if padding is set to 1. + The value to add to each response count if padding is `True`. Default value is 1/(2*nRatings) Returns @@ -63,14 +68,26 @@ def count_trials( Vectors containing the total number of responses in each accuracy category, conditional on presentation of S1 and S2. + Raises + ------ + ValueError + If `data` is neither `None` nor a DataFrame; if neither `responses` + nor `accuracy` is provided; if the inputs have different lengths; if + `responses` and `accuracy` disagree on a valid trial; or if no valid + trials remain. + Notes ----- - All trials where `stimuli` is not 0 or 1, accuracy is not 0 or 1, or confidence is - not in the range [1, nRatings], are automatically omitted. + All trials where `stimuli` is not 0 or 1, the response is not 0 or 1, or + confidence is not in the range [1, nRatings], are automatically omitted. + When `accuracy` is provided, trials where accuracy is not 0 or 1 (including + missing values) are omitted as well. - The inputs can be responses, accuracy or both. If both `responses` and - `accuracy` are provided, will check for consstency. If only `accuracy` is - provided, the responses vector will be automatically infered. + The inputs can be responses, accuracy or both. If only `accuracy` is + provided, the responses are inferred from it: a correct response equals the + stimulus, an incorrect one is the other stimulus. If both `responses` and + `accuracy` are provided, they are checked for consistency, and a + `ValueError` is raised on any trial where they disagree. If nR_S1 = [100 50 20 10 5 1], then when stimulus S1 was presented, the subject had the following accuracy counts:: @@ -95,13 +112,11 @@ def count_trials( Examples -------- - >>> stimID = [0, 1, 0, 0, 1, 1, 1, 1] - >>> accuracy = [0, 1, 1, 1, 0, 0, 1, 1] - >>> confidence = [1, 2, 3, 4, 4, 3, 2, 1] - >>> nRatings = 4 - - >>> nR_S1, nR_S2 = count_trials(data, nRatings=4) - >>> print(nR_S1, nR_S2) + >>> from cpm.utils.metad import count_trials + >>> count_trials(stimuli=[0, 1, 0, 0, 1, 1, 1, 1], + ... accuracy=[0, 1, 1, 1, 0, 0, 1, 1], + ... confidence=[1, 2, 3, 4, 4, 3, 2, 1], nRatings=4) + (array([1, 1, 0, 0, 1, 0, 0, 0]), array([1, 1, 0, 0, 1, 2, 0, 0])) References ---------- @@ -167,6 +182,15 @@ def count_trials( ) if accuracy is not None: valid &= binary_accuracy + mismatch = valid & ((responses == stimuli) != (accuracy == 1)) + if mismatch.any(): + positions = np.flatnonzero(mismatch) + raise ValueError( + f"`responses` and `accuracy` are inconsistent on {len(positions)} " + f"trial(s), at positions {positions[:5].tolist()}" + f"{' and more' if len(positions) > 5 else ''}: a response is " + "correct when it equals the stimulus." + ) if not valid.any(): raise ValueError( "count_trials found no valid trials: `stimuli` and responses must be 0 " diff --git a/test/applications/test_signal_detection.py b/test/applications/test_signal_detection.py index 91f29b5b..57049ea3 100644 --- a/test/applications/test_signal_detection.py +++ b/test/applications/test_signal_detection.py @@ -35,7 +35,7 @@ def synthetic_data(): "signal": [0, 1, 0, 1], "response": [0, 1, 0, 1], "confidence": [1, 2, 2, 1], - "accuracy": [1, 0, 1, 1], + "accuracy": [1, 1, 1, 1], "observed": [1, 0, 1, 0], }) diff --git a/test/utils/test_metad.py b/test/utils/test_metad.py index 09c2d4f2..cf96e59f 100644 --- a/test/utils/test_metad.py +++ b/test/utils/test_metad.py @@ -92,3 +92,34 @@ def test_padding_default_amount(): def test_non_dataframe_data_raises(): with pytest.raises(ValueError, match="DataFrame"): count_trials(data=[1, 2, 3]) + + +def test_consistent_responses_and_accuracy_match_responses_only(): + data = pd.DataFrame( + {"Stimuli": STIM, "Responses": RESP, "Accuracy": ACC, "Confidence": CONF} + ) + assert_expected(count_trials(data, nRatings=4)) + + +def test_inconsistent_responses_and_accuracy_raises(): + accuracy = ACC.copy() + accuracy[2] = 0 + data = pd.DataFrame( + {"Stimuli": STIM, "Responses": RESP, "Accuracy": accuracy, "Confidence": CONF} + ) + with pytest.raises(ValueError, match="inconsistent") as error: + count_trials(data, nRatings=4) + assert "1 trial" in str(error.value) + assert "[2]" in str(error.value) + + +def test_inconsistency_on_dropped_trial_ignored(): + data = pd.DataFrame( + { + "Stimuli": STIM + [1], + "Responses": RESP + [1], + "Accuracy": ACC + [0], + "Confidence": CONF + [9], + } + ) + assert_expected(count_trials(data, nRatings=4)) From 3bfd3bb86e2f9878a184379619f17504fb605911 Mon Sep 17 00:00:00 2001 From: lenarddome Date: Wed, 11 Nov 2026 11:33:31 +0100 Subject: [PATCH 3/4] fix(metad): warn on stimuli not coded 0/1, drop non-integer confidence, name participant (#106) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 ++- cpm/applications/signal_detection.py | 25 +++++++++++-------- cpm/utils/metad.py | 20 ++++++++++++--- test/applications/test_signal_detection.py | 14 +++++++++++ test/utils/test_metad.py | 29 +++++++++++++++++++++- 5 files changed, 76 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c603e279..d0df50cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- `cpm.utils.metad.count_trials` now infers responses from accuracy when only accuracy is given, checks that responses and accuracy agree when both are given, and accepts arrays without a DataFrame; it previously returned all-zero counts when the responses column was missing ([#106](https://github.com/DevComPsy/cpm/issues/106)) +- `cpm.utils.metad.count_trials` now infers responses from accuracy when only accuracy is given, checks that responses and accuracy agree when both are given, and accepts arrays without a DataFrame; it previously returned all-zero counts when the responses column was missing. It now warns when it omits stimuli not coded 0/1, and raises an error instead of returning all-zero counts when no valid trials remain ([#106](https://github.com/DevComPsy/cpm/issues/106)) +- `cpm.applications.signal_detection.EstimatorMetaD` names the participant when counting their trials fails ([#106](https://github.com/DevComPsy/cpm/issues/106)) ## [0.26.0] - 2026-10-16 diff --git a/cpm/applications/signal_detection.py b/cpm/applications/signal_detection.py index e1510827..16dacecc 100644 --- a/cpm/applications/signal_detection.py +++ b/cpm/applications/signal_detection.py @@ -556,16 +556,21 @@ def __task(participant, **args): nRatings = self.bins nCriteria = int(2 * nRatings - 1) # number criteria to be fitted - nR_S1, nR_S2 = count_trials( - data=subject, - stimuli="signal", - responses="response", - accuracy="accuracy", - confidence="discrete", - nRatings=nRatings, - padding=True, - padAmount=1/nRatings, - ) + try: + nR_S1, nR_S2 = count_trials( + data=subject, + stimuli="signal", + responses="response", + accuracy="accuracy", + confidence="discrete", + nRatings=nRatings, + padding=True, + padAmount=1/nRatings, + ) + except ValueError as e: + raise ValueError( + f"Error counting trials for participant {ppt}: {e}" + ) from e results_dict = fit_metad( nR_S1=nR_S1, diff --git a/cpm/utils/metad.py b/cpm/utils/metad.py index 25900047..b5772296 100644 --- a/cpm/utils/metad.py +++ b/cpm/utils/metad.py @@ -1,3 +1,5 @@ +import warnings + import numpy as np import pandas as pd @@ -79,9 +81,11 @@ def count_trials( Notes ----- All trials where `stimuli` is not 0 or 1, the response is not 0 or 1, or - confidence is not in the range [1, nRatings], are automatically omitted. - When `accuracy` is provided, trials where accuracy is not 0 or 1 (including - missing values) are omitted as well. + confidence is not a whole number in the range [1, nRatings], are + automatically omitted. When `accuracy` is provided, trials where accuracy + is not 0 or 1 (including missing values) are omitted as well. Stimuli must + be coded 0 for S1 and 1 for S2: a `UserWarning` is issued when stimuli + other than 0, 1 or missing values are omitted, for example with 1/2 coding. The inputs can be responses, accuracy or both. If only `accuracy` is provided, the responses are inferred from it: a correct response equals the @@ -163,6 +167,15 @@ def count_trials( ) binary_stimuli = np.isin(stimuli, (0, 1)) + out_of_range = ~binary_stimuli & ~np.isnan(stimuli) + if out_of_range.any(): + warnings.warn( + f"{out_of_range.sum()} trial(s) have stimuli outside {{0, 1}} (values " + f"{np.unique(stimuli[out_of_range])[:5].tolist()}) and are omitted; " + "code stimuli as 0 for S1 and 1 for S2.", + UserWarning, + stacklevel=2, + ) if accuracy is not None: binary_accuracy = np.isin(accuracy, (0, 1)) if responses is None: @@ -179,6 +192,7 @@ def count_trials( & np.isin(responses, (0, 1)) & (confidence >= 1) & (confidence <= nRatings) + & (confidence == np.round(confidence)) ) if accuracy is not None: valid &= binary_accuracy diff --git a/test/applications/test_signal_detection.py b/test/applications/test_signal_detection.py index 57049ea3..90046ec5 100644 --- a/test/applications/test_signal_detection.py +++ b/test/applications/test_signal_detection.py @@ -117,3 +117,17 @@ def test_metad_nll_equals_the_scipy_computation(): args = (nR_S1, nR_S2, n_ratings, rng.uniform(0.2, 2), rng.normal(0, 0.5), 1) assert signal_detection.metad_nll(guess, *args) == pytest.approx( _metad_nll_with_scipy(guess, *args), rel=1e-14) + + +def test_estimatormetad_names_participant_on_count_error(synthetic_data): + data = synthetic_data.copy() + data.loc[3, "accuracy"] = 0 # participant 2: signal 1, response 1, incorrect + est = signal_detection.EstimatorMetaD( + data=data.groupby("participant"), + bins=2, + parallel=False, + display=0, + ppt_identifier="participant", + ) + with pytest.raises(ValueError, match="participant 2.*inconsistent"): + est.optimise() diff --git a/test/utils/test_metad.py b/test/utils/test_metad.py index cf96e59f..388bffb8 100644 --- a/test/utils/test_metad.py +++ b/test/utils/test_metad.py @@ -1,3 +1,5 @@ +import warnings + import numpy as np import pandas as pd import pytest @@ -73,7 +75,9 @@ def test_no_valid_trials_raises(): data = pd.DataFrame( {"Stimuli": [s + 2 for s in STIM], "Accuracy": ACC, "Confidence": CONF} ) - with pytest.raises(ValueError, match="no valid trials"): + with pytest.raises(ValueError, match="no valid trials"), pytest.warns( + UserWarning, match="outside" + ): count_trials(data, nRatings=4) @@ -123,3 +127,26 @@ def test_inconsistency_on_dropped_trial_ignored(): } ) assert_expected(count_trials(data, nRatings=4)) + + +def test_stimuli_outside_0_1_warn(): + data = pd.DataFrame( + {"Stimuli": [s + 1 for s in STIM], "Accuracy": ACC, "Confidence": CONF} + ) + with pytest.warns(UserWarning, match=r"5 trial\(s\).*outside \{0, 1\}"): + count_trials(data, nRatings=4) + + +def test_valid_and_missing_stimuli_do_not_warn(): + data = pd.DataFrame( + {"Stimuli": STIM + [np.nan], "Accuracy": ACC + [1], "Confidence": CONF + [1]} + ) + with warnings.catch_warnings(): + warnings.simplefilter("error") + assert_expected(count_trials(data, nRatings=4)) + + +def test_non_integer_confidence_raises(): + data = pd.DataFrame({"Stimuli": STIM, "Accuracy": ACC, "Confidence": [2.5] * 8}) + with pytest.raises(ValueError, match="no valid trials"): + count_trials(data, nRatings=4) From 280b2d004f8a445dda79f871592848382dc5b139 Mon Sep 17 00:00:00 2001 From: lenarddome Date: Wed, 11 Nov 2026 12:09:50 +0100 Subject: [PATCH 4/4] chore(release): prepare 0.26.1 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- cpm/__version__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0df50cb..7b1bd7db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.26.1] - 2026-11-11 ### Fixed diff --git a/cpm/__version__.py b/cpm/__version__.py index 7c4a9591..025f4c5d 100644 --- a/cpm/__version__.py +++ b/cpm/__version__.py @@ -1 +1 @@ -__version__ = "0.26.0" +__version__ = "0.26.1"