diff --git a/.github/workflows/umi-v2.yml b/.github/workflows/umi-v2.yml index 8cf3daf5..2de5a61f 100644 --- a/.github/workflows/umi-v2.yml +++ b/.github/workflows/umi-v2.yml @@ -11,9 +11,9 @@ jobs: coordinated: name: UMI v2 # Update the workflow and runner revision together. - uses: KartikP/brainscore-unified/.github/workflows/offline-tests.yml@529237ada8c243cab0ad67cb53abde350342f4ea + uses: KartikP/brainscore-unified/.github/workflows/offline-tests.yml@5dc12f6654f5701b5135c7b288a3327ee704384e with: candidate: core candidate_repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} candidate_revision: ${{ github.event.pull_request.head.sha || github.sha }} - unified_revision: 529237ada8c243cab0ad67cb53abde350342f4ea + unified_revision: 5dc12f6654f5701b5135c7b288a3327ee704384e diff --git a/README.md b/README.md index 4c8cb456..a1dd78aa 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Brain-Score turns experimental data into quantitative benchmarks that evaluate subjects with compatible inputs, outputs and methods. > **UMI integration:** `Subject` defines session interaction through -> `interact(session)`. `BrainScoreModel` implements it with extraction and task +> `participate(session)`. `BrainScoreModel` implements it with extraction and task > helpers, including `process()`. Existing vision `BrainModel` and language > `ArtificialSubject` plugins remain supported through adapters. See > [the interface guide](docs/UMI_MIGRATION.md). diff --git a/brainscore_core/__init__.py b/brainscore_core/__init__.py index 5dc8c0cf..76c21ad1 100644 --- a/brainscore_core/__init__.py +++ b/brainscore_core/__init__.py @@ -32,7 +32,7 @@ from . import nulls # Warn loudly if the installed dependency versions have drifted off UMI's pins, -# since drift silently breaks/changes scoring (see _env_check for detail). -from ._env_check import warn_on_drift as _warn_on_drift +# since drift silently breaks/changes scoring (see environment_check for detail). +from .environment_check import warn_on_drift as _warn_on_drift _warn_on_drift() del _warn_on_drift diff --git a/brainscore_core/_env_check.py b/brainscore_core/_env_check.py index 0b4d813c..3487b6c3 100644 --- a/brainscore_core/_env_check.py +++ b/brainscore_core/_env_check.py @@ -1,122 +1,10 @@ -"""Runtime check that installed dependencies fall within UMI's compatibility bounds. +"""Former private name of :mod:`brainscore_core.environment_check`. -These are the version ranges scoring is *verified against* — deliberately looser -than any single install pin (that's the environment file's job). Drift outside -them silently breaks scoring. This module warns loudly at import rather than -letting a scored run be silently wrong. - -``scikit-learn`` moved from 1.5 to 1.7.2 on 2026-09-10, matching the version -``language/environment_lock.yml`` already locked. The 1.5 pin had made four -registered benchmarks unrunnable — ``Pereira2018.{243,384}sentences-ridge``, -``Blank2014-ridge`` and ``Fedorenko2016-ridge`` all route through -``kfold='group'``, and upstream calls ``GroupKFold(shuffle=..., random_state=...)``, -parameters that only exist from 1.6. - -The bump is score-neutral, which was checked rather than assumed: -``LogisticRegression(multi_class='multinomial')`` is still honoured in 1.7.2 and -returns bit-identical probabilities to 1.5.1 for both binary and multiclass -fits. sklearn's own warning says removal was planned for 1.7; it actually -slipped to **1.8**, which is why the upper bound is ``<1.8``. At 1.8 the -argument disappears and binary problems become proper binary logistic fits -(as if ``multi_class='ovr'``), which does move probabilities — measured on a -seeded 400x30 binary fit, mid-range values shift from 0.0019 to 0.0076. -Multiclass is unaffected. Crossing into 1.8 therefore needs the behavioral -readouts re-measured, and is a deliberate step rather than a version bump. - -``transformers`` 5 is now supported. The two things that blocked it are handled: -``DynamicCache.to_legacy_cache`` was removed, so the KV sliding window selects a -cache API by shape (``slice_kv_cache``), and the image-processor class names were -rebound so the default moved from PIL to torchvision, which registrations pin -explicitly (``hf_compat.pin_image_processor``). Both were checked against 5.14: -GPT-2 features are bit-identical across the two majors, and the one model whose -processor default actually moved shifts its score by 0.038%. - -Uses only ``importlib.metadata`` so ``brainscore_core`` stays free of heavy -dependencies (no torch/transformers import). +The old name is the same module object, not a copy, so code that patches +``_env_check.version`` (as older tests do) still changes what the checks see. """ -from importlib.metadata import version, PackageNotFoundError - -# (distribution name, ok(version_tuple) -> bool, human-readable bound) -_BOUNDS = [ - ("transformers", lambda v: (4, 57) <= (v[0], v[1]) < (6, 0), ">=4.57,<6"), - ("scikit-learn", lambda v: (1, 7) <= (v[0], v[1]) < (1, 8), ">=1.7,<1.8"), - ("numpy", lambda v: (1, 21) <= (v[0], v[1]) < (2, 0), ">=1.21,<2"), - ("xarray", lambda v: (v[0], v[1], v[2]) == (2022, 3, 0), "==2022.3.0"), -] - - -def _parse(s): - """Leading-numeric version tuple, padded to 3 fields. '5.5.0' -> (5, 5, 0).""" - parts = [] - for field in s.split(".")[:3]: - num = "" - for ch in field: - if ch.isdigit(): - num += ch - else: - break - parts.append(int(num) if num else 0) - while len(parts) < 3: - parts.append(0) - return tuple(parts) - - -def check_env_bounds(): - """Return human-readable drift messages (empty if all bounds hold).""" - drift = [] - for pkg, ok, expected in _BOUNDS: - try: - installed = version(pkg) - except PackageNotFoundError: - continue # package not installed in this (e.g. core-only) env - try: - if not ok(_parse(installed)): - drift.append( - f"{pkg}=={installed} is outside the UMI runtime " - f"compatibility bound ({expected})") - except Exception: - continue # unparseable version string: don't block import - return drift - - -def warn_on_drift(): - """Emit a RuntimeWarning if any dependency is outside its compatibility bound.""" - drift = check_env_bounds() - if drift: - import warnings - warnings.warn( - "Installed dependencies are outside UMI's runtime compatibility " - "bounds (these are the versions scoring is verified against; the exact " - "install pin is the environment file's job):\n - " - + "\n - ".join(drift) - + "\nScoring may be SILENTLY WRONG (scikit-learn>=1.8 drops " - "LogisticRegression(multi_class=...), so binary behavioral-readout " - "scores move without erroring). Install the pinned environment " - "(environment-unified.yml) to fix.", - RuntimeWarning, - stacklevel=2, - ) - - -def describe_env(): - """Per-dependency status, including the ones that are fine. +import sys - ``check_env_bounds`` reports only problems, which is what a warning needs. - A user asking "is my environment right?" needs to see the passing rows too, - otherwise silence is indistinguishable from nothing having been checked. +from . import environment_check - :return: rows of ``(package, installed or None, expected, ok)`` where ``ok`` - is None for a package that is absent or has an unparseable version - """ - rows = [] - for pkg, ok, expected in _BOUNDS: - try: - installed = version(pkg) - except PackageNotFoundError: - rows.append((pkg, None, expected, None)) - continue - try: - rows.append((pkg, installed, expected, bool(ok(_parse(installed))))) - except Exception: - rows.append((pkg, installed, expected, None)) - return rows +sys.modules[__name__] = environment_check diff --git a/brainscore_core/behavioral.py b/brainscore_core/behavioral.py index 9bf9fa24..aa764ed9 100644 --- a/brainscore_core/behavioral.py +++ b/brainscore_core/behavioral.py @@ -116,7 +116,8 @@ def start_task(self, task_context_or_task, fitting_stimuli=None, f"Model '{owner.identifier}' cannot run task " f"'{task_type}' via generation path " f"(prefer_path='generation'): " - f"requires model.generation_fn AND TaskContext.instruction " + f"requires an answer function registered with " + f"BrainScoreModel(answer=...) AND TaskContext.instruction " f"AND TaskContext.label_set to be set." ) self.use_generation_for_task = True @@ -155,8 +156,8 @@ def start_task(self, task_context_or_task, fitting_stimuli=None, raise ValueError( f"Model '{owner.identifier}' cannot run behavioral task " f"'{task_type}': TaskContext provides neither fitting_stimuli " - f"(needed for readout path) nor (instruction + label_set " - f"+ model.generation_fn) (needed for generation path)." + f"(needed for readout path) nor (instruction + label_set, " + f"with BrainScoreModel(answer=...)) (needed for generation path)." ) def reset(self) -> None: @@ -238,7 +239,7 @@ def predict_probabilities(self, stimuli): return self.readout_classifier.predict_proba(features) def generate_predictions(self, stimuli): - """Call generation_fn per-stimulus, return one-hot BehavioralAssembly.""" + """Call the registered answer function per stimulus, return one-hot BehavioralAssembly.""" import numpy as np from brainscore_core.supported_data_standards.brainio.assemblies import ( BehavioralAssembly, @@ -267,7 +268,7 @@ def generate_predictions(self, stimuli): if predicted not in label_to_idx: import warnings warnings.warn( - f"generation_fn returned {predicted!r}, not in " + f"answer function returned {predicted!r}, not in " f"label_set={label_set}. Defaulting to {label_set[0]!r}." ) predicted = label_set[0] diff --git a/brainscore_core/brainscore_model.py b/brainscore_core/brainscore_model.py index b7749694..061ae39d 100644 --- a/brainscore_core/brainscore_model.py +++ b/brainscore_core/brainscore_model.py @@ -22,10 +22,10 @@ from .selection import CompositeSelector, LayerSelector, UnitSelector, _promote_to_selector from .streaming import StreamEvent from .streaming_helpers import ( - _drain_stream_events as _drain_neural_stream_events, + drain_session_inputs as _drain_neural_stream_events, _reconstruct_stimulus_set_from_events as _reconstruct_neural_stimulus_set, _requested_output_channels, - _stimuli_from_stream_session as _neural_stimuli_from_stream_session, + session_stimuli as _neural_stimuli_from_stream_session, ) @@ -48,14 +48,14 @@ def _warn_old_names(**given) -> None: continue warnings.warn( f"BrainScoreModel({name}=...) is deprecated; use " - f"{_PREFERRED_NAMES[name]}. It keeps working until the public release, " - "when the old names are removed together with the interact-to-participate rename.", + f"{_PREFERRED_NAMES[name]}. It keeps working until the old names are " + "removed in a separate, announced release.", DeprecationWarning, stacklevel=3) class BrainScoreModel(UnifiedModel): """The one concrete subject, assembled from parts (extractors, region map, - answer, policy, intervention, capabilities). Evaluated through ``interact``; + answer, policy, intervention, capabilities). Evaluated through ``participate``; ``process`` is a typed convenience and the legacy benchmark methods forward to :class:`~brainscore_core.legacy_facade.LegacyBenchmarkFacade`.""" @@ -108,36 +108,38 @@ def __init__( """A model (or, in principle, a human) evaluated through the unified interface. Only ``identifier`` is always required; a feature-extraction model also - needs ``region_layer_map`` and an ``activations_model`` mapping. - Capability functions are optional; each enables another evaluation path. + needs ``region_layer_map`` and an ``extractor`` (a modality -> extractor + mapping). ``answer``, ``policy`` and ``intervention`` are optional; each + enables another evaluation path. :param identifier: unique model name (also the activation-cache key). :param model: the object perturbation hooks / generation run against (e.g. the torch module); may be ``None`` for a feature-extraction model whose wrapper holds its own module. :param region_layer_map: brain region -> layer path (or a ``UnitSelector``). - :param preprocessors: input preparation callables, declared by modality. - Configure each wrapper with its preparation function or tokenizer. - :param activations_model: extraction callable ``(stimuli, layers=...)``, - shared across declared modalities, or a modality -> extractor mapping. - Image, text, audio and video wrappers use this same slot. A mapping - declares its modalities without needing placeholder preprocessors. + :param preprocessors: compatibility parameter (deprecated). Prepare inputs + inside each extractor instead. + :param activations_model: compatibility parameter (deprecated); use + ``extractor``. :param visual_degrees: vision field-of-view; ignored by non-vision models. :param behavioral_readout_layer: layer on which to fit a behavioral readout. - :param extractor: extraction callable or modality-to-extractor mapping. - Prepare inputs inside each extractor. Preferred over activations_model. - :param answer: generate and parse a behavioral answer. - :param policy: choose an action from an EnvironmentStep. - :param intervention: apply a StateChange and return its cleanup function. - The return value is (PerturbationApplied, cleanup). - :param generation_fn: generate-and-parse callable for instruction-following - behavioral evaluation. - :param action_fn: ``EnvironmentStep -> EnvironmentResponse`` policy for - closed-loop / embodied evaluation. - :param state_change_fn: ``StateChange -> (PerturbationApplied, cleanup)`` for - lesion / stimulation evaluation. - :param supported_modalities: inputs accepted without an extraction wrapper, - for example by a closed-weight generation function. Extractor mappings + :param extractor: extraction callable ``(stimuli, layers=...)`` or a + modality -> extractor mapping. Image, text, audio and video wrappers + all use this slot; a mapping declares its input modalities. Prepare + inputs inside each extractor. + :param answer: ``(stimulus_row, instruction, label_set) -> label``; + generates and parses a behavioral answer for instruction-following + tasks. + :param policy: ``EnvironmentStep -> EnvironmentResponse``; chooses an + action for closed-loop / embodied evaluation. + :param intervention: ``StateChange -> (PerturbationApplied, cleanup)``; + applies a lesion / stimulation and returns its cleanup function. + :param generation_fn: compatibility parameter (deprecated); use ``answer``. + :param action_fn: compatibility parameter (deprecated); use ``policy``. + :param state_change_fn: compatibility parameter (deprecated); use + ``intervention``. + :param supported_modalities: inputs accepted without an extractor, for + example by a closed-weight ``answer`` function. Extractor mappings already declare their inputs; do not repeat them here. :param required_modalities: input modalities the model cannot run without. :param backbone_id: shared cache key so registrations with identical weights @@ -248,6 +250,16 @@ def capability_config(self): """Read-only registration configuration for external capability authors.""" return MappingProxyType(self._capability_config) + @property + def extractor(self) -> Mapping[str, Callable]: + """Read-only modality -> extractor mapping, as normalized at construction. + + Named like the ``extractor`` constructor argument; covers the older + ``activations_model``/``preprocessors`` forms too. For observation + tools that hook the wired modules; dispatch does not read it. + """ + return MappingProxyType(self._extractors) + def capability_state(self, identifier): """Per-model capability state, initialized by the registry before dispatch.""" return self._capability_state[identifier] @@ -472,12 +484,15 @@ def process(self, input_event: InputEvent, multi_modality: bool = False) -> OutputEvent: """Evaluate one input event and return the corresponding output event. - A typed convenience, not part of the subject contract (``interact`` is + A typed convenience, not part of the subject contract (``participate`` is the evaluation method). Dispatch is by ``input_event`` type: a ``StimulusSet`` / ``MultimodalStimulusSet`` -> ``NeuroidAssembly`` of recorded activations (call ``start_recording`` first); a ``StateChange`` -> ``PerturbationApplied`` (apply) or ``None`` (reset); an - ``EnvironmentStep`` -> ``EnvironmentResponse`` (needs ``action_fn``). + ``EnvironmentStep`` -> ``EnvironmentResponse`` (needs ``policy``). + Native benchmarks evaluate subjects through sessions or the typed + helpers in ``brainscore_core.streaming_helpers``; ``process`` is not + available on every subject. :param input_event: the stimulus / state-change / environment-step to run. :param multi_modality: if True, extract from every detected-and-supported @@ -494,7 +509,7 @@ def process(self, input_event: InputEvent, output = repeat_over_time_bins(output, time_bins) return output - def interact(self, session) -> None: + def participate(self, session) -> None: """Run a session through the one registered capability that claims it. The handler sees only the streams this model declares in @@ -505,7 +520,7 @@ def interact(self, session) -> None: from .capabilities.registry import _setup_once from .session_view import DeclaredInputSession _setup_once(self, handler) - handler.interact(self, DeclaredInputSession(session, self.in_channels)) + handler.participate(self, DeclaredInputSession(session, self.in_channels)) def _handle_environment_step(self, input_event: 'EnvironmentStep') -> OutputEvent: return self._dispatcher.handle_environment_step(input_event) @@ -517,9 +532,13 @@ def _handle_stimulus_set(self, stimuli, multi_modality: bool = False) -> OutputEvent: return self._dispatcher.handle_stimulus_set(stimuli, multi_modality) - def _detect_modalities(self, stimuli) -> Set[str]: + def detect_modalities(self, stimuli) -> Set[str]: + """Declared input modalities whose columns appear in ``stimuli``.""" return self._dispatcher.detect_modalities(stimuli) + # Compatibility alias; internal callers still use it. + _detect_modalities = detect_modalities + def _pick_modality(self, detected: Set[str]) -> str: return self._dispatcher.pick_modality(detected) @@ -658,7 +677,7 @@ def _generate_predictions(self, stimuli): # --- Legacy benchmark surface ------------------------------------------ # Permanent: legacy benchmarks are never removed. Behavior lives in # LegacyBenchmarkFacade, which runs each legacy call as a session through - # ``interact``; these forwards seed a facade with this model's current + # ``participate``; these forwards seed a facade with this model's current # recording and task state so direct calls keep working. @property diff --git a/brainscore_core/capabilities/base.py b/brainscore_core/capabilities/base.py index 1ee081f0..1037eb18 100644 --- a/brainscore_core/capabilities/base.py +++ b/brainscore_core/capabilities/base.py @@ -35,13 +35,17 @@ def supports_session(self, model, channels) -> bool: """Opt into a complete requested output combination before execution.""" return False - def interact(self, model, session) -> None: + def participate(self, model, session) -> None: """Execute a session accepted by supports_session.""" raise NotImplementedError(self.identifier) def setup(self, model) -> Dict[str, Any]: - """Return per-model state stored under ``model._capability_state``.""" + """Return this capability's initial per-model state. + + The registry stores it before dispatch; read it back in ``handles`` / + ``process`` with ``model.capability_state(self.identifier)``. + """ del model return {} diff --git a/brainscore_core/capabilities/embodied_action.py b/brainscore_core/capabilities/embodied_action.py index ef22ecdb..50b36f29 100644 --- a/brainscore_core/capabilities/embodied_action.py +++ b/brainscore_core/capabilities/embodied_action.py @@ -7,7 +7,7 @@ class EmbodiedActionCapability(Capability): - """Dispatch EnvironmentStep and Message events through ``action_fn``.""" + """Dispatch EnvironmentStep and Message events through the model's ``policy``.""" identifier = 'embodied-action' order = 300 @@ -51,10 +51,10 @@ def _process_environment_step(self, model, input_event: EnvironmentStep): ) if model._action_fn is None: raise NotImplementedError( - f"Model '{model.identifier}' has no action_fn registered. " - f"Embodied evaluation requires the model to declare an " - f"action_fn(env_step) -> EnvironmentResponse callable at " - f"BrainScoreModel construction time. Received: " + f"Model '{model.identifier}' has no policy (no action_fn registered " + f"under the compatibility name). Embodied evaluation requires " + f"BrainScoreModel(policy=...), a callable " + f"policy(env_step) -> EnvironmentResponse. Received: " f"EnvironmentStep(step_num={input_event.step_num})." ) response = model._action_fn(input_event) @@ -70,10 +70,10 @@ def _process_environment_step(self, model, input_event: EnvironmentStep): def _process_message(self, model, input_event: Message): if model._action_fn is None: raise NotImplementedError( - f"Model '{model.identifier}' has no action_fn registered; " - f"process(Message) routes to the agent's responder. Declare " - f"action_fn(env_step) -> EnvironmentResponse|Message at " - f"BrainScoreModel construction time." + f"Model '{model.identifier}' has no policy (no action_fn registered " + f"under the compatibility name); a Message routes to the agent's " + f"policy. Pass BrainScoreModel(policy=...), a callable " + f"policy(env_step) -> EnvironmentResponse|Message." ) response = model._action_fn(EnvironmentStep(observation=input_event)) if not isinstance(response, (Message, EnvironmentResponse)): diff --git a/brainscore_core/capabilities/sessions.py b/brainscore_core/capabilities/sessions.py index ec001f4a..10464825 100644 --- a/brainscore_core/capabilities/sessions.py +++ b/brainscore_core/capabilities/sessions.py @@ -3,7 +3,7 @@ Each part of a BrainScoreModel (region map, answer or readout, policy, intervention) handles some channels. These capabilities declare those channels and run the matching sessions, registered the same way as any added -capability, so ``interact``, ``check_session_support`` and the declared +capability, so ``participate``, ``check_session_support`` and the declared channels all read one registry. They sort after added capabilities (``order`` 10000), so an added capability that claims the same outputs wins. """ @@ -40,12 +40,11 @@ def declared_output_channels(self, model): def supports_session(self, model, channels) -> bool: return bool(channels) and all(c.startswith('neural:') for c in channels) - def interact(self, model, session) -> None: + def participate(self, model, session) -> None: if getattr(session, 'streaming', False): _drivers._drive_neural_session_streaming(model, session) else: - _drivers._drive_neural_session_via_process(model, session, - allow_multi_modality=True) + _drivers.drive_neural_session(model, session, allow_multi_modality=True) class BehaviorSessions(_SessionCapability): @@ -64,11 +63,11 @@ def declared_output_channels(self, model): def supports_session(self, model, channels) -> bool: return set(channels) == {'behavior'} - def interact(self, model, session) -> None: + def participate(self, model, session) -> None: if getattr(session, 'streaming', False): _drivers._drive_behavior_session_streaming(model, session) else: - _drivers._drive_behavior_session_via_process(model, session) + _drivers.drive_behavior_session(model, session) class MotorSessions(_SessionCapability): @@ -86,7 +85,7 @@ def declared_output_channels(self, model): def supports_session(self, model, channels) -> bool: return set(channels) == {'motor'} - def interact(self, model, session) -> None: + def participate(self, model, session) -> None: _drivers._drive_environment_session_via_process(model, session) @@ -105,7 +104,7 @@ def declared_output_channels(self, model): def supports_session(self, model, channels) -> bool: return set(channels) == {'perturbation'} - def interact(self, model, session) -> None: + def participate(self, model, session) -> None: _drivers._drive_state_change_session_via_process(model, session) diff --git a/brainscore_core/contract.py b/brainscore_core/contract.py index 3119512a..5640f66b 100644 --- a/brainscore_core/contract.py +++ b/brainscore_core/contract.py @@ -20,17 +20,19 @@ class TaskContext: Benchmarks populate all relevant fields. Models consume whichever fields match their paradigm: - Feature models use fitting_stimuli + label_set to train a readout - - Instruction-following models use instruction + label_set to build a prompt + (``BrainScoreModel(behavioral_readout_layer=...)``) + - Instruction-following models use instruction + label_set with their + ``answer`` function (``BrainScoreModel(answer=...)``) - Models that support both can choose their preferred approach ``prefer_path`` lets a benchmark (or an experimenting user) pin which behavioral dispatch path is selected when a model supports more than one. Accepts ``'auto'`` (default — generation if available, else - readout), ``'generation'`` (force generation, error if generation_fn - is missing or instruction is empty), or ``'readout'`` (force readout, - error if behavioral_readout_layer is missing or fitting_stimuli is - empty). This is the public API replacement for the legacy monkey-patch - pattern of setting ``model._generation_fn = None`` to force readout. + readout), ``'generation'`` (force generation through the model's + ``answer``; error if no answer is registered or instruction/label_set + is empty), or ``'readout'`` (force readout, error if + behavioral_readout_layer is missing or fitting_stimuli is empty). Use + it instead of disabling a model's answer function to force readout. """ task_type: str label_set: Optional[List[str]] = None @@ -44,7 +46,7 @@ class TaskContext: class Subject(ABC): """Minimal contract for a subject evaluated through a session. - Declare identity and supported channels, then implement ``interact`` to + Declare identity and supported channels, then implement ``participate`` to consume inputs with ``session.next_input()`` and submit responses with ``session.respond(event)``. A subject need not expose layers, modalities, or legacy model methods. Declarations may be properties or class attributes. @@ -77,7 +79,7 @@ def required_channels(self) -> Set[str]: return set() @abstractmethod - def interact(self, session: "Session") -> None: + def participate(self, session: "Session") -> None: """Consume session inputs and emit responses until the session ends.""" ... @@ -90,7 +92,7 @@ class UnifiedModel(Subject): """Subject base for models evaluated through typed paths. Implementations provide ``process``, ``start_task``, ``start_recording``, - a layer map and modality declarations. ``interact`` (the evaluation + a layer map and modality declarations. ``participate`` (the evaluation method of the contract) runs a session through those typed paths, so a typed model needs no session code of its own. ``process`` is a typed convenience of this base, not part of the subject contract. @@ -154,12 +156,12 @@ def required_channels(self) -> Set[str]: return modalities_to_input_channels(self.required_modalities) def process(self, input_event: InputEvent) -> OutputEvent: - """Typed convenience used by the default ``interact``; not part of the contract.""" + """Typed convenience used by the default ``participate``; not part of the contract.""" raise NotImplementedError( - f"{type(self).__name__} defines no process(); implement interact(session) " + f"{type(self).__name__} defines no process(); implement participate(session) " "or process(input_event) for the typed session driver.") - def interact(self, session: "Session") -> None: + def participate(self, session: "Session") -> None: """Run ``session`` through this model's typed paths.""" from .streaming_helpers import drive_typed_session drive_typed_session(self, session) diff --git a/brainscore_core/dispatch.py b/brainscore_core/dispatch.py index 289513b4..4f362ddc 100644 --- a/brainscore_core/dispatch.py +++ b/brainscore_core/dispatch.py @@ -64,9 +64,11 @@ def register_input_handler(cls, event_type: type, handler_name: str, def normalize_extractors(owner, preprocessors, activations_model): """Adapt registration forms once to ``extractor(stimuli, layers=...)``. - Explicit extractors never use attribute-based capability discovery. Older - registrations that put extraction functions in preprocessors are adapted by - their callable signature at this compatibility boundary. + ``activations_model`` here is the normalized ``extractor`` argument (the + compatibility spelling is the same slot). Explicit extractors never use + attribute-based capability discovery. Older registrations that put + extraction functions in preprocessors are adapted by their callable + signature at this compatibility boundary. """ from collections.abc import Mapping import inspect @@ -75,12 +77,16 @@ def normalize_extractors(owner, preprocessors, activations_model): extractors = dict(activations_model) elif activations_model is not None: if not preprocessors: - raise ValueError('A shared activations_model requires declared preprocessors/modalities.') + raise ValueError( + 'A single shared extractor needs its input modalities declared. ' + 'Pass extractor={modality: extractor} instead.') extractors = {key: activations_model for key in preprocessors} else: extractors = {} if any(not callable(value) for value in extractors.values()): - raise TypeError('Every activations_model extractor must be callable.') + raise TypeError( + 'Every extractor must be callable as extractor(stimuli, layers=...). ' + 'Check the values passed in extractor={modality: extractor}.') layer_extractors = set(extractors) for modality, preprocessor in preprocessors.items(): if modality in extractors: @@ -95,7 +101,9 @@ def normalize_extractors(owner, preprocessors, activations_model): except (TypeError, ValueError) as error: if modality in extractors: continue - raise TypeError('Declare this callable explicitly in activations_model.') from error + raise TypeError( + 'Cannot inspect this preprocessor. Declare it explicitly as ' + 'extractor={modality: extractor}.') from error variadic_wrapper = (not inspect.isfunction(preprocessor) and bool(signature.parameters) and all( p.kind in (p.VAR_POSITIONAL, p.VAR_KEYWORD) @@ -115,14 +123,14 @@ def normalize_extractors(owner, preprocessors, activations_model): warnings.warn( 'Extraction callbacks in preprocessors are deprecated and are removed at ' 'the public release. Use ' - 'activations_model={modality: extractor}, where extractor accepts ' + 'extractor={modality: extractor}, where extractor accepts ' '(stimuli, layers=...).', DeprecationWarning, stacklevel=3) def extract(stimuli, *, layers, function=preprocessor): if len(layers) > 1: raise ValueError( 'This compatibility extraction callback accepts one layer. ' - 'For multiple layers, use activations_model={modality: ' + 'For multiple layers, use extractor={modality: ' 'extractor} with an extractor accepting (stimuli, layers=...).') return function(owner._model, stimuli, recording_layer=layers[0] if layers else None) @@ -246,8 +254,8 @@ def _extract_for_modality_inner(self, stimuli, modality: str, layers: List[str]) return extractor(stimuli, layers=layers) raise NotImplementedError( f"Model '{owner.identifier}' declares {modality!r} input but has no " - 'activation extractor for it. Configure activations_model to record ' - 'internal activity, or use a behavioral task for an output-only model.') + 'activation extractor for it. Pass extractor={modality: extractor} to ' + 'record internal activity, or use a behavioral task for an output-only model.') def process_multi_modality(self, stimuli, detected: Set[str], layers: List[str]): @@ -280,11 +288,12 @@ def process_multi_modality(self, stimuli, detected: Set[str], owner._extract_for_modality(stimuli, modality, modality_layers)) if not hasattr(sub, 'dims') or 'neuroid' not in sub.dims: raise TypeError( - f"multi_modality dispatch requires every preprocessor " + f"multi_modality dispatch requires every extractor " f"to return an xarray-shaped assembly with a 'neuroid' " f"dim; modality '{modality}' returned " - f"{type(sub).__name__}. Upgrade the preprocessor to a " - f"layer-aware extractor (TextWrapper / VLMVisionWrapper)." + f"{type(sub).__name__}. Pass a layer-aware extractor " + f"(TextWrapper / VLMVisionWrapper) via " + f"extractor={{modality: extractor}}." ) if owner._recorder.is_multi_region and not owner._recorder.composite_recording: sub = owner._tag_neuroids_with_regions(sub, modality) diff --git a/brainscore_core/environment_check.py b/brainscore_core/environment_check.py new file mode 100644 index 00000000..d19fbddf --- /dev/null +++ b/brainscore_core/environment_check.py @@ -0,0 +1,128 @@ +"""Runtime check that installed dependencies fall within UMI's compatibility bounds. + +Public API: ``check_env_bounds`` (drift messages only), ``describe_env`` (one +row per checked package, passing or not) and ``warn_on_drift`` (the warning +``import brainscore_core`` emits). ``python -m brainscore.doctor`` prints +``describe_env`` as a table. + +These are the version ranges scoring is *verified against* — deliberately looser +than any single install pin (that's the environment file's job). Drift outside +them silently breaks scoring. This module warns loudly at import rather than +letting a scored run be silently wrong. + +``scikit-learn`` moved from 1.5 to 1.7.2 on 2026-09-10, matching the version +``language/environment_lock.yml`` already locked. The 1.5 pin had made four +registered benchmarks unrunnable — ``Pereira2018.{243,384}sentences-ridge``, +``Blank2014-ridge`` and ``Fedorenko2016-ridge`` all route through +``kfold='group'``, and upstream calls ``GroupKFold(shuffle=..., random_state=...)``, +parameters that only exist from 1.6. + +The bump is score-neutral, which was checked rather than assumed: +``LogisticRegression(multi_class='multinomial')`` is still honoured in 1.7.2 and +returns bit-identical probabilities to 1.5.1 for both binary and multiclass +fits. sklearn's own warning says removal was planned for 1.7; it actually +slipped to **1.8**, which is why the upper bound is ``<1.8``. At 1.8 the +argument disappears and binary problems become proper binary logistic fits +(as if ``multi_class='ovr'``), which does move probabilities — measured on a +seeded 400x30 binary fit, mid-range values shift from 0.0019 to 0.0076. +Multiclass is unaffected. Crossing into 1.8 therefore needs the behavioral +readouts re-measured, and is a deliberate step rather than a version bump. + +``transformers`` 5 is now supported. The two things that blocked it are handled: +``DynamicCache.to_legacy_cache`` was removed, so the KV sliding window selects a +cache API by shape (``slice_kv_cache``), and the image-processor class names were +rebound so the default moved from PIL to torchvision, which registrations pin +explicitly (``hf_compat.pin_image_processor``). Both were checked against 5.14: +GPT-2 features are bit-identical across the two majors, and the one model whose +processor default actually moved shifts its score by 0.038%. + +Uses only ``importlib.metadata`` so ``brainscore_core`` stays free of heavy +dependencies (no torch/transformers import). +""" +from importlib.metadata import version, PackageNotFoundError +from typing import List, Optional, Tuple + +# (distribution name, ok(version_tuple) -> bool, human-readable bound) +_BOUNDS = [ + ("transformers", lambda v: (4, 57) <= (v[0], v[1]) < (6, 0), ">=4.57,<6"), + ("scikit-learn", lambda v: (1, 7) <= (v[0], v[1]) < (1, 8), ">=1.7,<1.8"), + ("numpy", lambda v: (1, 21) <= (v[0], v[1]) < (2, 0), ">=1.21,<2"), + ("xarray", lambda v: (v[0], v[1], v[2]) == (2022, 3, 0), "==2022.3.0"), +] + + +def _parse(s): + """Leading-numeric version tuple, padded to 3 fields. '5.5.0' -> (5, 5, 0).""" + parts = [] + for field in s.split(".")[:3]: + num = "" + for ch in field: + if ch.isdigit(): + num += ch + else: + break + parts.append(int(num) if num else 0) + while len(parts) < 3: + parts.append(0) + return tuple(parts) + + +def check_env_bounds() -> List[str]: + """Return human-readable drift messages (empty if all bounds hold).""" + drift = [] + for pkg, ok, expected in _BOUNDS: + try: + installed = version(pkg) + except PackageNotFoundError: + continue # package not installed in this (e.g. core-only) env + try: + if not ok(_parse(installed)): + drift.append( + f"{pkg}=={installed} is outside the UMI runtime " + f"compatibility bound ({expected})") + except Exception: + continue # unparseable version string: don't block import + return drift + + +def warn_on_drift() -> None: + """Emit a RuntimeWarning if any dependency is outside its compatibility bound.""" + drift = check_env_bounds() + if drift: + import warnings + warnings.warn( + "Installed dependencies are outside UMI's runtime compatibility " + "bounds (these are the versions scoring is verified against; the exact " + "install pin is the environment file's job):\n - " + + "\n - ".join(drift) + + "\nScoring may be SILENTLY WRONG (scikit-learn>=1.8 drops " + "LogisticRegression(multi_class=...), so binary behavioral-readout " + "scores move without erroring). Install the pinned environment " + "(environment-unified.yml) to fix.", + RuntimeWarning, + stacklevel=2, + ) + + +def describe_env() -> List[Tuple[str, Optional[str], str, Optional[bool]]]: + """Per-dependency status, including the ones that are fine. + + ``check_env_bounds`` reports only problems, which is what a warning needs. + A user asking "is my environment right?" needs to see the passing rows too, + otherwise silence is indistinguishable from nothing having been checked. + + :return: rows of ``(package, installed or None, expected, ok)`` where ``ok`` + is None for a package that is absent or has an unparseable version + """ + rows = [] + for pkg, ok, expected in _BOUNDS: + try: + installed = version(pkg) + except PackageNotFoundError: + rows.append((pkg, None, expected, None)) + continue + try: + rows.append((pkg, installed, expected, bool(ok(_parse(installed))))) + except Exception: + rows.append((pkg, installed, expected, None)) + return rows diff --git a/brainscore_core/events.py b/brainscore_core/events.py index b7d8e353..0f212cb3 100644 --- a/brainscore_core/events.py +++ b/brainscore_core/events.py @@ -21,8 +21,8 @@ class Selection: :param layer: Dotted module path within the model, e.g. ``'language_model.layers.20'`` or ``'visual.transformer.blocks.10'``. - Resolved by the registered ``state_change_fn`` against the concrete - model object. A path names the module's output; add ``':input'`` to + Resolved by the model's registered ``intervention`` against the + concrete model object. A path names the module's output; add ``':input'`` to name its first input instead, e.g. ``'model.layers.14.mlp.down_proj:input'`` for a transformer's MLP neurons, which are computed inline and output by no module. Recording @@ -36,7 +36,7 @@ class Selection: :param expected_shape: Optional non-batch tensor shape, checked before use. Localizers set this when coordinates describe a flattened output. :param metadata: Free-form bag for selection criteria the - ``state_change_fn`` can interpret (e.g., a localizer mask, a + ``intervention`` can interpret (e.g., a localizer mask, a functional contrast specification). """ layer: str @@ -63,7 +63,7 @@ class Perturbation: ``scale``), ``'drive'`` (add ``amount`` — a stimulation / excitation analog), ``'replace'`` (replace with ``replacement`` tensor), or model-specific kinds (e.g., ``'noise'``) that the registered - ``state_change_fn`` knows how to interpret. + ``intervention`` knows how to interpret. :param scale: Multiplier for ``kind='scale'``. Ignored otherwise. :param amount: Additive offset for ``kind='drive'`` — inject activity into the selected units, the counterpart of ablation. A number, or (with @@ -83,15 +83,16 @@ class StateChange: """Apply or remove an intervention in the model. Used to simulate conditions like dyslexia, prosopagnosia, or pharmacological - effects. Passed to ``model.process()`` the same way stimuli are — the - interface is generic over input event type so benchmarks studying neural + effects. A session (or the typed ``apply_state_change`` helper) delivers it + to the subject; ``BrainScoreModel.process()`` also accepts it as a typed + convenience, the same way it accepts stimuli. Benchmarks studying neural dysfunction or drug effects do not need a separate model method. Two roles, distinguished by ``kind``: 1. **Apply** a perturbation: ``kind`` names a perturbation class (``'ablation'``, ``'lesion'``, ``'pharmacological'``, ...). The - ``state_change_fn`` registered on the model installs the perturbation + ``intervention`` registered on the model installs the perturbation (e.g., a forward hook), and ``process(StateChange)`` returns a :class:`PerturbationApplied` confirmation containing a ``handle_id``. 2. **Remove** a previously-applied perturbation: ``kind='reset'`` and @@ -187,9 +188,10 @@ class EnvironmentStep: """Single step of an embodied environment. Used for interactive evaluation where the model observes, acts, and receives - the environment's response over multiple ticks. Passed to ``process()`` like - a stimulus — the interface generalizes over input type rather than adding - a separate agent method. + the environment's response over multiple ticks. A session delivers it to + the subject like a stimulus (``BrainScoreModel.process()`` accepts it as a + typed convenience); the model's ``policy`` answers it. The interface + generalizes over input type rather than adding a separate agent method. This is a device-agnostic envelope. The preferred field is ``observation`` (an environment-defined payload). The ``cameras`` and @@ -204,8 +206,8 @@ class EnvironmentStep: Episode control signals (``is_first``, ``is_last``, ``is_terminal``, ``reward``, ``discount``) follow the RLDS convention so existing - DROID-format demonstration data can be replayed through ``process()`` - one step at a time. + DROID-format demonstration data can be replayed through a session one + step at a time. Examples:: @@ -299,8 +301,8 @@ class Message: the next agent's stimulus, with no bespoke routing protocol and no ABC change. - ``process(Message)`` routes to the model's ``action_fn`` (an agent - *responds* to a message), and a responder may return either a ``Message`` or + A ``Message`` routes to the model's ``policy`` (an agent *responds* to a + message), and a responder may return either a ``Message`` or an :class:`EnvironmentResponse`. ``content`` is the payload (text, an action, a structured proposal); ``sender`` / ``recipient`` carry addressing for multi-party settings; ``metadata`` carries telemetry without growing the diff --git a/brainscore_core/legacy_facade.py b/brainscore_core/legacy_facade.py index 9b206339..926216fb 100644 --- a/brainscore_core/legacy_facade.py +++ b/brainscore_core/legacy_facade.py @@ -6,9 +6,9 @@ this facade is permanent (Contract Rules v3, rule 11.2). It records each legacy setting, presents the stimuli as a ``BufferedSession`` that carries those settings (time bins, trial count, task, recording type) and runs the -session through the subject's ``interact``. Nothing here depends on what kind +session through the subject's ``participate``. Nothing here depends on what kind of subject is wrapped: a ``BrainScoreModel``, a domain adapter and an -interact-only ``Subject`` all take part the same way. +participate-only ``Subject`` all take part the same way. ``subject_for_benchmark`` applies the facade at the scoring boundary to benchmarks a domain loader marked as legacy. ``BrainScoreModel`` forwards its @@ -26,13 +26,13 @@ LEGACY_API = 'legacy' # calls look_at / digest_text and friends TYPED_API = 'typed' # calls start_recording + process on the candidate -SESSION_API = 'session' # declares channels and drives interact itself +SESSION_API = 'session' # declares channels and drives participate itself class LegacyBenchmarkFacade: """Present a subject as a legacy ``BrainModel`` / ``ArtificialSubject``. - :param subject: any subject with ``interact``, ``reset``, ``identifier`` + :param subject: any subject with ``participate``, ``reset``, ``identifier`` and ``out_channels``. :param visual_degrees: field of view reported to vision benchmarks, or ``None`` when the subject declares none. @@ -180,7 +180,7 @@ def _measure(self, stimuli, channels: List[str], settings: Dict[str, Any]): task_context=self._task_context(), time_bins=self._time_bins) session.settings = dict(settings) - self._subject.interact(session) + self._subject.participate(session) return {channel: session.collect(channel) for channel in channels} diff --git a/brainscore_core/memory.py b/brainscore_core/memory.py index c4aefe1d..6424704b 100644 --- a/brainscore_core/memory.py +++ b/brainscore_core/memory.py @@ -448,7 +448,7 @@ def _probe_feature_dim(model, benchmark, stimulus_set, else _probe_regions(model, benchmark)) last_error = None for region in regions: - # One session through interact, the same way every subject is + # One session through participate, the same way every subject is # evaluated; no probing of the subject's methods or class. try: if region is None: diff --git a/brainscore_core/streaming_helpers.py b/brainscore_core/streaming_helpers.py index 4dc2071a..fb3c4dae 100644 --- a/brainscore_core/streaming_helpers.py +++ b/brainscore_core/streaming_helpers.py @@ -12,7 +12,7 @@ import numpy as np -from .contract import Subject, UnifiedModel # noqa: F401 (re-exported) +from .contract import Subject, TaskContext, UnifiedModel # noqa: F401 (re-exported) from .io_catalog import modalities_to_input_channels from .streaming import BufferedSession, Session, StreamEvent, parse_channel from .events import ( @@ -369,13 +369,13 @@ def neural_response(subject, stimulus_set, record: Union[str, Sequence[str]] = " "neural_response returns a single assembly and therefore takes a single " f"region; got {record!r}. To record several regions, use the session API:\n" " session = stimulus_session(stimuli, record=['V4', 'IT'])\n" - " subject.interact(session)\n" + " subject.participate(session)\n" " v4, it = session.collect('neural:V4'), session.collect('neural:IT')\n" "Or record them in one pass and split on the region coord:\n" " subject.start_recording(['V4', 'IT']); assembly = subject.process(stimuli)") session = stimulus_session(stimulus_set, record=record, time_bins=time_bins) - subject.interact(session) + subject.participate(session) return session.collect(f"neural:{record}") @@ -384,13 +384,13 @@ def behavioral_response(subject, task_context) -> BehavioralAssembly: ``BehavioralAssembly`` -- the model's output, not a Score. """ session = behavior_session(task_context) - subject.interact(session) + subject.participate(session) return session.collect("behavior") def apply_state_change(subject, state_change: StateChange): session = state_change_session(state_change) - subject.interact(session) + subject.participate(session) return session.collect("perturbation") @@ -412,7 +412,7 @@ def run_environment(subject, environment, *, seed=None, options=None, close() raise with session: - subject.interact(session) + subject.participate(session) if not session.complete: raise RuntimeError("Environment session ended before episode completion") return session.collect("motor") @@ -420,7 +420,7 @@ def run_environment(subject, environment, *, seed=None, options=None, def _drive_via_process(subject, session: StimulusSetSession, stimulus_set, record: Union[str, Sequence[str]] = "IT", time_bins=None) -> None: - """Interim Phase 2 bridge; Phase 3 swaps this for native interact().""" + """Interim Phase 2 bridge; Phase 3 swaps this for native participate().""" _start_recording(subject, record, time_bins=time_bins) output = subject.process(stimulus_set) session.emit(StreamEvent( @@ -431,18 +431,31 @@ def _drive_via_process(subject, session: StimulusSetSession, stimulus_set, )) -def _drive_neural_session_via_process( - subject, session, driver: str = "interact", allow_multi_modality: bool = False +# The ``driver`` values below ("interact", "interact_streaming") are stable +# record-format labels written into response meta, not callable API names. They +# predate the participate rename and stay as-is so saved records still compare +# equal under compare_outputs. +def drive_neural_session( + subject: Any, session: Any, driver: str = "interact", + allow_multi_modality: bool = False, ) -> None: - """Run a neural session through ``start_recording`` + ``process``. - - ``allow_multi_modality`` is the subject's own statement that its - ``process`` combines several input modalities; when several are present - it is called with ``multi_modality=True``. + """Answer a session's ``neural:`` requests with one ``process`` call. + + For typed subjects such as legacy adapters and ``BrainScoreModel``, which + already implement ``start_recording(target, time_bins=...)`` and + ``process(stimuli)``. The contract does not require ``process``: an + arbitrary ``Subject`` only implements ``participate`` and never needs this. + + Reads every input from ``session``, records all requested regions in one + pass using the session's ``time_bins``, and responds once per requested + channel. A request for trial-to-trial variance in the session settings is + refused before any compute. ``allow_multi_modality`` is the subject's own + statement that its ``process`` combines several input modalities; when + several are present it is called with ``multi_modality=True``. """ _refuse_variance(subject, session) - input_events = _drain_stream_events(session) - stimuli = _stimuli_from_stream_session(session, input_events) + input_events = drain_session_inputs(session) + stimuli = session_stimuli(session, input_events) output_t_ms = input_events[-1].t_ms if input_events else 0.0 use_multi_modality = allow_multi_modality and _spans_several_inputs(input_events) time_bins = getattr(session, "time_bins", None) @@ -452,7 +465,7 @@ def _drive_neural_session_via_process( family, region = parse_channel(channel) if family != "neural" or region is None: raise NotImplementedError( - "neural interact currently supports requested " + "neural participate currently supports requested " f"neural: output channels; got {channel!r}." ) channel_regions.append((channel, region)) @@ -577,7 +590,7 @@ def _drive_neural_session_streaming( family, region = parse_channel(channel) if family != "neural" or region is None: raise NotImplementedError( - "streaming neural interact supports requested neural: " + "streaming neural participate supports requested neural: " f"output channels; got {channel!r}." ) channel_regions.append((channel, region)) @@ -591,7 +604,7 @@ def _drive_neural_session_streaming( while event is not None: if not isinstance(event, StreamEvent): raise TypeError( - "streaming neural interact expects StreamEvent inputs; got " + "streaming neural participate expects StreamEvent inputs; got " f"{type(event).__name__}." ) stimuli = _one_row_stimuli(session, event, stream_index) @@ -655,7 +668,13 @@ def _preserve_stimulus_set_type(original, sliced): return sliced -def _drain_stream_events(session) -> list[StreamEvent]: +def drain_session_inputs(session: Any) -> list[StreamEvent]: + """Read every input event from ``session`` until it returns ``None``. + + Events come back in the order the session sent them. Anything that is not + a ``StreamEvent`` raises ``TypeError``. Use this when a subject answers the + whole session at once rather than one input at a time. + """ events: list[StreamEvent] = [] while True: event = session.next_input() @@ -663,13 +682,20 @@ def _drain_stream_events(session) -> list[StreamEvent]: return events if not isinstance(event, StreamEvent): raise TypeError( - "neural interact expects StreamEvent inputs; got " + "neural participate expects StreamEvent inputs; got " f"{type(event).__name__}." ) events.append(event) -def _stimuli_from_stream_session(session, events: list[StreamEvent]): +def session_stimuli(session: Any, events: list[StreamEvent]) -> Any: + """The stimulus table a session presented. + + Returns the session's own ``stimulus_set`` object unchanged when it has + one, so presentation metadata (``object_name`` and friends) survives. When + it has none, rebuilds a minimal ``StimulusSet`` from ``events`` that keeps + only the input columns and ``stimulus_id``. + """ if getattr(session, "stimulus_set", None) is not None: return session.stimulus_set return _reconstruct_stimulus_set_from_events(events) @@ -694,7 +720,7 @@ def _reconstruct_stimulus_set_from_events(events: list[StreamEvent]): """ if not events: raise ValueError( - "neural interact received no input events and no session.stimulus_set " + "neural participate received no input events and no session.stimulus_set " "context to reconstruct." ) @@ -762,7 +788,7 @@ def _requested_output_channels(session) -> list[str]: return list(_neural_channels(record)) raise ValueError( - "streaming interact requires the session to declare " + "streaming participate requires the session to declare " "requested_output_channels or record." ) @@ -779,7 +805,7 @@ def drive_typed_session(subject, session) -> None: if streaming: _drive_behavior_session_streaming(subject, session) else: - _drive_behavior_session_via_process(subject, session) + drive_behavior_session(subject, session) return if requested_channels == ["perturbation"]: _drive_state_change_session_via_process(subject, session) @@ -794,7 +820,7 @@ def drive_typed_session(subject, session) -> None: if streaming: _drive_neural_session_streaming(subject, session) else: - _drive_neural_session_via_process(subject, session) + drive_neural_session(subject, session) return raise NotImplementedError( f"Subject {getattr(subject, 'identifier', type(subject).__name__)!r} " @@ -830,7 +856,7 @@ def _drive_environment_session_via_process( def _drive_state_change_via_process(subject, session: StateChangeSession, state_change: StateChange) -> None: - """Interim Phase 2 bridge; Phase 3 swaps this for native interact().""" + """Interim Phase 2 bridge; Phase 3 swaps this for native participate().""" _drive_state_change_session_via_process( subject, session, state_change=state_change, driver="process" ) @@ -846,12 +872,12 @@ def _drive_state_change_session_via_process( if event is not None: if not isinstance(event, StreamEvent): raise TypeError( - "perturbation interact expects StreamEvent inputs; got " + "perturbation participate expects StreamEvent inputs; got " f"{type(event).__name__}." ) if not isinstance(event.payload, StateChange): raise TypeError( - "perturbation interact expects StateChange payloads; got " + "perturbation participate expects StateChange payloads; got " f"{type(event.payload).__name__}." ) state_change = event.payload @@ -859,7 +885,7 @@ def _drive_state_change_session_via_process( state_change = getattr(session, "state_change", None) if state_change is None: raise ValueError( - "perturbation interact requires a StateChange input event." + "perturbation participate requires a StateChange input event." ) result = subject.process(state_change) handle_id = _state_change_handle_id(state_change, result) @@ -877,17 +903,17 @@ def _drive_state_change_session_via_process( def _drive_behavior_via_process(subject, session: BehavioralSession, task_context) -> None: - """Interim Phase 2 bridge; Phase 3 swaps this for native interact().""" - _drive_behavior_session_via_process( + """Interim Phase 2 bridge; Phase 3 swaps this for native participate().""" + drive_behavior_session( subject, session, task_context=task_context, driver="process" ) def _behavior_stimuli_from_session(session, task_context): - events = [event for event in _drain_stream_events(session) + events = [event for event in drain_session_inputs(session) if event.channel != 'instruction' and event.meta.get('role') != 'fitting'] if events or getattr(session, 'stimulus_set', None) is not None: - return _stimuli_from_stream_session(session, events) + return session_stimuli(session, events) return _behavior_stimuli_to_score(task_context) @@ -913,16 +939,28 @@ def _refuse_variance(subject, session) -> None: "add it with microsaccades; this subject has no equivalent.") -def _drive_behavior_session_via_process( - subject, session: BehavioralSession, task_context=None, - driver: str = "interact" +def drive_behavior_session( + subject: Any, session: Any, task_context: Optional[TaskContext] = None, + driver: str = "interact", ) -> None: - """Drive a behavioral session through existing start_task/process logic.""" + """Answer a session's ``behavior`` request with ``start_task`` then one ``process`` call. + + For typed subjects such as legacy adapters and ``BrainScoreModel``, which + already implement ``start_task(task_context)`` and ``process(stimuli)``. + The contract does not require ``process``: an arbitrary ``Subject`` only + implements ``participate`` and never needs this. + + Uses ``task_context`` when given, otherwise the session's. Fitting inputs + and instructions are not scored; the scored trials are the session's + ``stimulus_set``, or the task's own stimuli when the session sends none. + A request for trial-to-trial variance is refused before any compute. The + single response is sent on the ``behavior`` channel. + """ if task_context is None: task_context = getattr(session, "task_context", None) if task_context is None: raise ValueError( - "behavior interact requires a session.task_context to drive." + "behavior participate requires a session.task_context to drive." ) _refuse_variance(subject, session) subject.start_task(task_context) @@ -941,6 +979,14 @@ def _drive_behavior_session_via_process( )) +# Former private names, kept for existing callers. Each is the same function +# object as its public name. +_drain_stream_events = drain_session_inputs +_stimuli_from_stream_session = session_stimuli +_drive_neural_session_via_process = drive_neural_session +_drive_behavior_session_via_process = drive_behavior_session + + def _drive_behavior_session_streaming( subject, session: BehavioralSession, task_context=None, driver: str = "interact_streaming" @@ -960,7 +1006,7 @@ def _drive_behavior_session_streaming( task_context = getattr(session, "task_context", None) if task_context is None: raise ValueError( - "behavior interact requires a session.task_context to drive." + "behavior participate requires a session.task_context to drive." ) subject.start_task(task_context) # once, before any trial stimuli = _behavior_stimuli_from_session(session, task_context) @@ -989,7 +1035,7 @@ class StreamingStimulusSetSession(Session): row iterator, so event buffering stays bounded. Truly generated feeds that do not already exist as a table use ``WindowedStreamSession`` instead. - ``streaming = True`` is what routes ``interact`` to the one-at-a-time driver. + ``streaming = True`` is what routes ``participate`` to the one-at-a-time driver. """ streaming = True diff --git a/brainscore_core/submission/database.py b/brainscore_core/submission/database.py index bfe5d19f..f275a8be 100644 --- a/brainscore_core/submission/database.py +++ b/brainscore_core/submission/database.py @@ -363,7 +363,7 @@ def benchmarkinstance_from_benchmark(benchmark: Benchmark, domain: str) -> Bench # the version has changed and the benchmark instance was not yet in the database ceiling = benchmark.ceiling bench_inst.ceiling = ceiling.item() - bench_inst.ceiling_error = _retrieve_score_error(ceiling) + bench_inst.ceiling_error = retrieve_score_error(ceiling) bench_inst.save() return bench_inst @@ -394,12 +394,12 @@ def update_score(score: ScoreObject, entry: Score): require_leaderboard_eligible(score.attrs.get('leaderboard_eligible', True)) if 'ceiling' not in score.attrs: # many engineering benchmarks do not have a primate ceiling # only store raw (unceiled) value - entry.score_raw = _retrieve_score_center(score) + entry.score_raw = retrieve_score_center(score) else: # score has a ceiling. Store ceiled as well as raw value - score_raw = _retrieve_score_center(score.raw) + score_raw = retrieve_score_center(score.raw) entry.score_raw = score_raw - entry.score_ceiled = _retrieve_score_center(score) - entry.error = _retrieve_score_error(score) + entry.score_ceiled = retrieve_score_center(score) + entry.error = retrieve_score_error(score) if 'comment' in score.attrs: entry.comment = score.attrs['comment'] logger.debug(f"updating raw score: {entry.score_raw}, ceiled score: {entry.score_ceiled}, error: {entry.error}, " @@ -407,8 +407,10 @@ def update_score(score: ScoreObject, entry: Score): entry.save() -def _retrieve_score_center(score: ScoreObject) -> float: +def retrieve_score_center(score: ScoreObject) -> float: """ + The score value the leaderboard stores. + Deal with multiple formats of storing the score, i.e. a single scalar in the score object versus an aggregation dimension. """ @@ -417,8 +419,10 @@ def _retrieve_score_center(score: ScoreObject) -> float: return score.item() -def _retrieve_score_error(score: ScoreObject) -> Union[float, None]: +def retrieve_score_error(score: ScoreObject) -> Union[float, None]: """ + The score error the leaderboard stores. + Deal with multiple formats of storing the score, i.e. an error attribute in the score object versus an aggregation dimension. Returns None if no error was found @@ -428,3 +432,8 @@ def _retrieve_score_error(score: ScoreObject) -> Union[float, None]: if 'error' in score.attrs: return score.attrs['error'] return None + + +# Former private names, kept for existing callers (same function objects). +_retrieve_score_center = retrieve_score_center +_retrieve_score_error = retrieve_score_error diff --git a/docs/UMI_MIGRATION.md b/docs/UMI_MIGRATION.md index 92f4885d..fc6c99a4 100644 --- a/docs/UMI_MIGRATION.md +++ b/docs/UMI_MIGRATION.md @@ -1,13 +1,16 @@ # Migrating core concepts to UMI `Subject` is the session contract. A native subject declares `identifier`, -`in_channels`, and `out_channels`, and implements `interact(session)`. +`in_channels`, and `out_channels`, and implements `participate(session)`. `required_channels` defaults to an empty set. Override `reset()` when the subject -holds state that must be cleared between independent evaluations. +holds state that must be cleared between independent evaluations. A `behavior` +response is a `BehavioralAssembly` labeled by `stimulus_id`, so a benchmark can +match each answer to the stimulus that produced it. ```python from brainscore_core import Subject -from brainscore_core.streaming import InMemorySession, StreamEvent +from brainscore_core.streaming import BufferedSession, StreamEvent +from brainscore_core.supported_data_standards.brainio.assemblies import BehavioralAssembly class EchoSubject(Subject): identifier = "echo" @@ -15,26 +18,40 @@ class EchoSubject(Subject): out_channels = {"behavior"} required_channels = {"text"} - def interact(self, session): + def participate(self, session): while (event := session.next_input()) is not None: - session.emit(StreamEvent( - "behavior", event.payload, event.t_ms, dict(event.meta))) + answer = BehavioralAssembly( + [event.payload], + coords={"stimulus_id": ("presentation", [event.meta["stimulus_id"]]), + "sentence": ("presentation", [event.payload])}, + dims=["presentation"]) + session.respond(StreamEvent( + channel="behavior", payload=answer, + t_ms=event.t_ms, meta=dict(event.meta))) subject = EchoSubject() -session = InMemorySession([StreamEvent("text", "hello", 0.0)]) -subject.interact(session) -assert session.emitted[0].payload == "hello" +session = BufferedSession( + inputs=[StreamEvent(channel="text", payload="hello", t_ms=0.0, + meta={"stimulus_id": "s1"})], + requested_output_channels=["behavior"]) +subject.participate(session=session) +response = session.emitted[0].payload +assert response["stimulus_id"].values.tolist() == ["s1"] +assert response["sentence"].values.tolist() == ["hello"] +assert response.values.tolist() == ["hello"] subject.reset() ``` A native subject does not need `process()`, `start_recording()`, `start_task()`, a layer map, modality properties, or `check_session_support()`. Scoring preflight uses channel declarations. A custom session only needs `next_input()` and -`emit()`; collection belongs to the session or the benchmark helper. +`respond()`; collection belongs to the session or the benchmark helper. +(`InMemorySession` and `emit()` are accepted compatibility spellings.) For table-based evaluations, `neural_response` and `behavioral_response` in `brainscore_core.streaming_helpers` build sessions and collect assemblies. The -benchmark applies its metric to those responses. +benchmark applies its metric to those responses. `apply_state_change` and +`run_environment` do the same for interventions and environment episodes. ## Choose helpers for your subject @@ -47,9 +64,32 @@ configurable implementation with extraction helpers. the vision and language adapters inherit this compatibility base, so they remain subjects and keep their existing computation paths. -Implement `Subject` directly when you want to own session handling. A benchmark -that calls `process()` or recording methods needs a subject providing those -methods; inheriting `Subject` alone does not provide them. +Implement `Subject` directly when you want to own session handling. Benchmarks +reach any subject the same way: + +- A native benchmark uses the typed helpers above or drives a session itself, + so it works with any subject; it does not need `process()` or recording + methods on the subject. +- An existing legacy benchmark (one that calls `look_at`, `digest_text`, + `start_recording` and similar) is scored through the permanent legacy + benchmark facade. `score()` wraps the subject in it automatically, and the + facade runs each legacy call as a session through `participate`. Legacy + benchmarks are never removed, so this path does not go away. + +For a `BrainScoreModel`, configure the parts by name: `extractor=` for +recorded activity, `answer=` for instruction-following behavior, `policy=` +for environment actions, and `intervention=` for lesions or stimulation. + +Writing an adapter for a model that already has those typed methods? Its +`participate(session)` can use the public functions in +`brainscore_core.streaming_helpers`. `drain_session_inputs(session)` reads +every input event in order. `session_stimuli(session, events)` returns the +stimulus table the session presented. `drive_neural_session(subject, session)` +and `drive_behavior_session(subject, session)` answer a `neural:` or +`behavior` request with the subject's own `start_recording` or `start_task` +followed by one `process` call. These functions exist only for subjects that +have those methods. A subject written directly against `Subject` never needs +them. Existing vision `BrainModel` and language `ArtificialSubject` plugins keep working through their permanent adapters. No plugin removal or re-registration diff --git a/docs/source/modules/glossary.rst b/docs/source/modules/glossary.rst index 9978ddfa..aa467706 100644 --- a/docs/source/modules/glossary.rst +++ b/docs/source/modules/glossary.rst @@ -29,7 +29,7 @@ Glossary Subject (UMI) The interface through which an experiment or benchmark uses a model. - A native Subject declares its channels and implements interact(session). + A native Subject declares its channels and implements participate(session). BrainScoreModel implements Subject with extraction and task helpers. ArtificialSubject diff --git a/tests/test_api_cohesion.py b/tests/test_api_cohesion.py index caeb975a..767923ee 100644 --- a/tests/test_api_cohesion.py +++ b/tests/test_api_cohesion.py @@ -83,7 +83,7 @@ def test_common_session_collects_neural_output_without_helper_class(): activations_model={'vision': Extractor()}) session = InMemorySession(stimulus_set=StimulusSet({'image_path': ['x']}), requested_output_channels=('neural:IT',)) - model.interact(session) + model.participate(session) assert session.collect().shape == (1, 4) @@ -142,7 +142,7 @@ def test_neural_events_work_without_a_stimulus_table(): activations_model={'text': Extractor()}) session = InMemorySession(inputs=[StreamEvent(channel='text', payload='hello', t_ms=0)], requested_output_channels=['neural:language']) - model.interact(session) + model.participate(session) assert session.collect().shape == (1, 4) @@ -160,7 +160,7 @@ def test_common_session_collects_batch_and_streamed_assemblies(streaming): requested_output_channels=['neural:language'], streaming=streaming, ) - model.interact(session) + model.participate(session) assert session.collect().shape == (2, 4) diff --git a/tests/test_channel_compatibility.py b/tests/test_channel_compatibility.py index b2959c4a..89ab7090 100644 --- a/tests/test_channel_compatibility.py +++ b/tests/test_channel_compatibility.py @@ -39,7 +39,7 @@ def out_channels(self): def required_channels(self): return set(self._required_channels) - def interact(self, session): + def participate(self, session): self.process_called = True diff --git a/tests/test_contract_drift.py b/tests/test_contract_drift.py index aecdc217..0751f561 100644 --- a/tests/test_contract_drift.py +++ b/tests/test_contract_drift.py @@ -38,7 +38,7 @@ class _MinimalSubject(Subject): in_channels = set() out_channels = set() - def interact(self, session): + def participate(self, session): pass @@ -94,16 +94,16 @@ def test_session_abc_contract_is_minimal(): assert callable(Session.collect) -def test_subject_interact_and_reset_contract_are_pinned(): - assert hasattr(Subject, "interact") +def test_subject_participate_and_reset_contract_are_pinned(): + assert hasattr(Subject, "participate") assert hasattr(Subject, "reset") assert Subject.__abstractmethods__ == { - "identifier", "in_channels", "out_channels", "interact", + "identifier", "in_channels", "out_channels", "participate", } assert "reset" not in Subject.__abstractmethods__ subject = _MinimalSubject() - assert subject.interact(session=None) is None + assert subject.participate(session=None) is None assert not hasattr(subject, "process") assert not hasattr(subject, "region_layer_map") assert not hasattr(subject, "supported_modalities") diff --git a/tests/test_env_check.py b/tests/test_env_check.py index bd203c3f..76d3aa0c 100644 --- a/tests/test_env_check.py +++ b/tests/test_env_check.py @@ -1,4 +1,4 @@ -"""Tests for the dependency-pin drift check (_env_check).""" +"""Tests for the dependency-pin drift check (environment_check, formerly _env_check).""" from brainscore_core import _env_check @@ -66,3 +66,71 @@ def test_transformers_just_below_shipped_floor_flagged(monkeypatch): "numpy": "1.26.4", "xarray": "2022.3.0"} monkeypatch.setattr(_env_check, "version", lambda p: v[p]) assert any("transformers" in d for d in _env_check.check_env_bounds()) + + +def test_public_module_is_the_old_module(): + """The old private name is an alias of the public module, not a copy.""" + from brainscore_core import environment_check + assert _env_check is environment_check + assert environment_check.describe_env is _env_check.describe_env + + +def test_patching_the_old_name_reaches_the_public_functions(monkeypatch): + from brainscore_core import environment_check + v = {"transformers": "4.57.1", "scikit-learn": "1.8.0", + "numpy": "1.26.4", "xarray": "2022.3.0"} + monkeypatch.setattr(_env_check, "version", lambda p: v[p]) + assert [row[0] for row in environment_check.describe_env() + if row[3] is False] == ["scikit-learn"] + + +def test_describe_env_reports_passing_absent_and_drifted_rows(monkeypatch): + from importlib.metadata import PackageNotFoundError + from brainscore_core import environment_check + v = {"transformers": "4.57.1", "numpy": "2.0.0", "xarray": "2022.3.0"} + + def _version(package): + if package not in v: + raise PackageNotFoundError(package) + return v[package] + monkeypatch.setattr(environment_check, "version", _version) + assert environment_check.describe_env() == [ + ("transformers", "4.57.1", ">=4.57,<6", True), + ("scikit-learn", None, ">=1.7,<1.8", None), + ("numpy", "2.0.0", ">=1.21,<2", False), + ("xarray", "2022.3.0", "==2022.3.0", True), + ] + + +def test_warn_on_drift_warns_with_the_drift(monkeypatch): + import pytest + from brainscore_core import environment_check + monkeypatch.setattr(environment_check, "check_env_bounds", + lambda: ["numpy==2.0.0 is outside"]) + with pytest.warns(RuntimeWarning, match="numpy==2.0.0 is outside"): + environment_check.warn_on_drift() + + +def test_environment_specs_module_is_separate(): + """brainscore_core.environment keeps observation/action specs only.""" + from brainscore_core import environment, environment_check + assert environment is not environment_check + assert not hasattr(environment, "describe_env") + + +def test_core_imports_without_torch_or_transformers(): + """Startup runs the drift check without importing heavy dependencies.""" + import subprocess + import sys + code = ( + "import sys\n" + "class _Block:\n" + " def find_spec(self, name, path=None, target=None):\n" + " if name.split('.')[0] in ('torch', 'transformers'):\n" + " raise ImportError('blocked ' + name)\n" + "sys.meta_path.insert(0, _Block())\n" + "import brainscore_core, brainscore_core.environment_check\n" + "import brainscore_core.streaming_helpers\n" + "assert 'torch' not in sys.modules and 'transformers' not in sys.modules\n" + ) + subprocess.run([sys.executable, "-c", code], check=True) diff --git a/tests/test_environment_contract.py b/tests/test_environment_contract.py index ea40bacf..c4f17198 100644 --- a/tests/test_environment_contract.py +++ b/tests/test_environment_contract.py @@ -70,7 +70,7 @@ def _typed_subject(process): """A minimal typed subject: process() plus the shared typed session driver.""" from brainscore_core.streaming_helpers import drive_typed_session subject = SimpleNamespace(process=process) - subject.interact = lambda session: drive_typed_session(subject, session) + subject.participate = lambda session: drive_typed_session(subject, session) return subject diff --git a/tests/test_external_extensions.py b/tests/test_external_extensions.py index 1cb2e405..4d8cdc03 100644 --- a/tests/test_external_extensions.py +++ b/tests/test_external_extensions.py @@ -37,7 +37,7 @@ def process(self, model, event, **kwargs): return StreamEvent('test_result', state['calls'] + event.payload, event.t_ms) def supports_session(self, model, channels): return set(channels) == {'test_result'} - def interact(self, model, session): + def participate(self, model, session): while (event := session.next_input()) is not None: session.emit(model.process(event)) register_capability(Tool()) diff --git a/tests/test_lab_vocabulary.py b/tests/test_lab_vocabulary.py index 58c6e4da..3b46f2c2 100644 --- a/tests/test_lab_vocabulary.py +++ b/tests/test_lab_vocabulary.py @@ -98,7 +98,7 @@ def extract(stimuli, *, layers): region_layer_map={'region': 'layer'}) session = BufferedSession(stimulus_set=StimulusSet({column: ['one', 'two']}), requested_output_channels=('neural:region',)) - subject.interact(session) + subject.participate(session) np.testing.assert_array_equal(session.collect().values, np.ones((2, 2))) @@ -137,3 +137,97 @@ def respond(self, event): getattr(session, spelling)(event) assert session.collect() == [event] assert session.calls == session.new_calls == 1 + + +# -- Human-facing guidance names the canonical parameters -------------------- + +def test_shared_extractor_without_modalities_names_extractor_mapping(): + with pytest.raises(ValueError, match=r'extractor=\{modality: extractor\}') as error: + BrainScoreModel(identifier='shared', extractor=lambda stimuli, layers: None) + assert 'activations_model' not in str(error.value) + + +def test_non_callable_extractor_names_extractor_signature(): + with pytest.raises(TypeError, match=r'extractor\(stimuli, layers=\.\.\.\)') as error: + BrainScoreModel(identifier='bad', extractor={'vision': 'not callable'}) + assert 'activations_model' not in str(error.value) + + +def _text_task(**fields): + from brainscore_core.model_interface import TaskContext + return TaskContext(task_type='label', **fields) + + +def test_forced_generation_without_answer_names_answer_parameter(): + subject = BrainScoreModel(identifier='no-answer', supported_modalities={'text'}) + task = _text_task(instruction='choose', label_set=['yes', 'no'], prefer_path='generation') + with pytest.raises(ValueError, match=r'BrainScoreModel\(answer=\.\.\.\)') as error: + subject.start_task(task) + assert 'generation_fn' not in str(error.value) + + +def test_unviable_behavioral_task_names_answer_parameter(): + subject = BrainScoreModel(identifier='no-path', supported_modalities={'text'}) + with pytest.raises(ValueError, match='neither fitting_stimuli') as error: + subject.start_task(_text_task()) + assert 'BrainScoreModel(answer=...)' in str(error.value) + assert 'generation_fn' not in str(error.value) + + +def test_deprecated_extraction_callback_warning_points_to_extractor(): + def callback(model, stimuli, recording_layer=None): + return None + with pytest.warns(DeprecationWarning) as record: + BrainScoreModel(identifier='callback', preprocessors={'vision': callback}) + messages = [str(w.message) for w in record if 'Extraction callbacks' in str(w.message)] + assert len(messages) == 1 + assert 'extractor={modality: extractor}' in messages[0] + assert 'activations_model' not in messages[0] + + +# -- Core guide teaching example --------------------------------------------- + +def test_guide_echo_subject_fixture_returns_labeled_behavioral_assembly(): + """Self-contained copy of the UMI_MIGRATION.md EchoSubject (no docs needed).""" + from brainscore_core import Subject + from brainscore_core.supported_data_standards.brainio.assemblies import BehavioralAssembly + + class EchoSubject(Subject): + identifier = 'echo' + in_channels = {'text'} + out_channels = {'behavior'} + required_channels = {'text'} + + def participate(self, session): + while (event := session.next_input()) is not None: + answer = BehavioralAssembly( + [event.payload], + coords={'stimulus_id': ('presentation', [event.meta['stimulus_id']]), + 'sentence': ('presentation', [event.payload])}, + dims=['presentation']) + session.respond(StreamEvent(channel='behavior', payload=answer, + t_ms=event.t_ms, meta=dict(event.meta))) + + session = BufferedSession( + inputs=[StreamEvent(channel='text', payload='hello', t_ms=0.0, + meta={'stimulus_id': 's1'})], + requested_output_channels=['behavior']) + EchoSubject().participate(session=session) + (response,) = [event.payload for event in session.emitted] + assert isinstance(response, BehavioralAssembly) + assert response['stimulus_id'].values.tolist() == ['s1'] + assert response['sentence'].values.tolist() == ['hello'] + assert response.values.tolist() == ['hello'] + + +# -- Missing policy names the canonical parameter ---------------------------- + +@pytest.mark.parametrize('event', [EnvironmentStep(), 'message']) +def test_missing_policy_names_policy_parameter(event): + from brainscore_core.events import Message + if event == 'message': + event = Message(content='hello') + subject = BrainScoreModel(identifier='no-policy') + with pytest.raises(NotImplementedError, match=r'BrainScoreModel\(policy=\.\.\.\)') as error: + subject.process(event) + assert 'has no policy' in str(error.value) diff --git a/tests/test_leaderboard_eligibility.py b/tests/test_leaderboard_eligibility.py index 67e5a925..8d0fb209 100644 --- a/tests/test_leaderboard_eligibility.py +++ b/tests/test_leaderboard_eligibility.py @@ -102,3 +102,17 @@ def test_submission_checks_returned_score_before_upload(monkeypatch, excluded): endpoint._score_model_on_benchmark('model', 'benchmark', None, 'vision', True, None) assert entry.score_raw == 0.25 endpoint._upload_score_to_s3.assert_called_once_with(score, 'model', 'benchmark', 'vision') + + +def test_public_score_retrieval_keeps_the_private_names_and_reductions(): + assert database._retrieve_score_center is database.retrieve_score_center + assert database._retrieve_score_error is database.retrieve_score_error + scalar = Score(0.25) + assert database.retrieve_score_center(scalar) == 0.25 + assert database.retrieve_score_error(scalar) is None + scalar.attrs['error'] = 0.01 + assert database.retrieve_score_error(scalar) == 0.01 + aggregated = Score([0.5, 0.02], coords={'aggregation': ['center', 'error']}, + dims=['aggregation']) + assert database.retrieve_score_center(aggregated) == 0.5 + assert database.retrieve_score_error(aggregated) == 0.02 diff --git a/tests/test_legacy_facade.py b/tests/test_legacy_facade.py index a8f5807e..8d51d454 100644 --- a/tests/test_legacy_facade.py +++ b/tests/test_legacy_facade.py @@ -172,7 +172,7 @@ def test_preferred_names_do_not_warn(): intervention=lambda c: (None, lambda: None)) -# --- the facade runs any subject through interact ------------------------ +# --- the facade runs any subject through participate ------------------------ from brainscore_core.model_interface import Subject from brainscore_core.streaming import StreamEvent @@ -180,7 +180,7 @@ def test_preferred_names_do_not_warn(): class _Recorder(Subject): - """An interact-only subject: no process, look_at or start_recording.""" + """A participate-only subject: no process, look_at or start_recording.""" identifier = 'recording-subject' in_channels = {'text', 'vision'} @@ -191,7 +191,7 @@ def __init__(self): self.sessions = [] self.resets = 0 - def interact(self, session): + def participate(self, session): self.sessions.append(session) while session.next_input() is not None: pass @@ -207,7 +207,7 @@ def reset(self): self.resets += 1 -def test_facade_runs_an_interact_only_subject_on_vision_calls(): +def test_facade_runs_an_participate_only_subject_on_vision_calls(): subject = _Recorder() facade = LegacyBenchmarkFacade(subject, visual_degrees=12) facade.start_recording('IT', time_bins=[(70, 170), (170, 270)]) @@ -282,11 +282,11 @@ def test_facade_reset_clears_settings_and_resets_subject(): facade.look_at(_images()) -def test_typed_look_at_runs_through_interact(): +def test_typed_look_at_runs_through_participate(): model, extractor = _model() sessions = [] - original = model.interact - model.interact = lambda session: (sessions.append(session), original(session))[1] + original = model.participate + model.participate = lambda session: (sessions.append(session), original(session))[1] model.start_recording('IT', time_bins=[(70, 170)]) via_legacy = model.look_at(_images()) assert len(sessions) == 1 diff --git a/tests/test_memory.py b/tests/test_memory.py index aad0dd3f..1feb3866 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -837,8 +837,8 @@ def test_unsliceable_stimulus_set_is_not_renamed(self): assert _isolate_probe_identifier(parent, parent).identifier == 'hvm-public' -class TestProbeThroughInteract: - """The feature-width probe runs one session through interact for every subject.""" +class TestProbeThroughParticipate: + """The feature-width probe runs one session through participate for every subject.""" def _stimuli(self): import pandas as pd @@ -848,7 +848,7 @@ def _stimuli(self): stimuli.identifier = 'probe-images' return stimuli - def test_interact_only_subject_is_probed_from_its_declarations(self): + def test_participate_only_subject_is_probed_from_its_declarations(self): from types import SimpleNamespace import numpy as np from brainscore_core.memory import _probe_feature_dim, _probe_regions @@ -862,7 +862,7 @@ class Native(Subject): out_channels = {'neural:IT'} required_channels = set() - def interact(self, session): + def participate(self, session): n = 0 while session.next_input() is not None: n += 1 diff --git a/tests/test_model_interface_public_api.py b/tests/test_model_interface_public_api.py index eb9fac9a..32369ba6 100644 --- a/tests/test_model_interface_public_api.py +++ b/tests/test_model_interface_public_api.py @@ -69,3 +69,124 @@ def test_brainscore_model_dispatch_order_snapshot(): ('EnvironmentStep', '_handle_environment_step'), ('Message', '_handle_message'), ] + + +# -- Public observation surface: ``extractor`` + ``detect_modalities`` -------- +# Core-only fakes (no framework). Observation tools read these instead of the +# private ``_extractors`` / ``_detect_modalities``; neither is a Subject member. + +import subprocess +import sys +from types import MappingProxyType + +import pytest + +from brainscore_core.model_interface import BrainScoreModel, Subject + + +class _Columns: + def __init__(self, columns): + self.columns = columns + + +class _FakeExtractor: + """Extractor with the layer-aware signature and an episode reset.""" + + def __init__(self, output): + self.output, self.calls, self.resets = output, [], 0 + + def __call__(self, stimuli, layers): + self.calls.append((stimuli, list(layers))) + return self.output + + def reset(self): + self.resets += 1 + + +def _forms(): + vision, text = _FakeExtractor('v-out'), _FakeExtractor('t-out') + return { + 'extractor': (BrainScoreModel('m', extractor={'vision': vision, 'text': text}), + {'vision': vision, 'text': text}), + 'activations_model_mapping': ( + BrainScoreModel('m', activations_model={'vision': vision, 'text': text}), + {'vision': vision, 'text': text}), + 'shared_activations_model': ( + BrainScoreModel('m', preprocessors={'vision': lambda x: x}, + activations_model=vision), + {'vision': vision}), + } + + +@pytest.mark.parametrize('form', ['extractor', 'activations_model_mapping', + 'shared_activations_model']) +def test_extractor_is_read_only_normalized_mapping(form): + model, expected = _forms()[form] + mapping = model.extractor + assert isinstance(mapping, MappingProxyType) + assert set(mapping) == set(expected) + assert all(mapping[k] is expected[k] for k in expected) # same objects, no copies + with pytest.raises(TypeError): + mapping['audio'] = _FakeExtractor('a') + with pytest.raises(TypeError): + del mapping['vision'] + with pytest.raises(AttributeError): + model.extractor = {} + assert dict(model.extractor) == dict(mapping) # unchanged by attempts + + +def test_extractor_is_not_a_subject_member(): + assert not hasattr(Subject, 'extractor') + assert not hasattr(Subject, 'detect_modalities') + + +def test_canonical_and_old_forms_declare_same_channels(): + forms = _forms() + new, old = forms['extractor'][0], forms['activations_model_mapping'][0] + before = (new.in_channels, new.out_channels, new.supported_modalities) + new.extractor # reading the mapping must not change declarations + assert (new.in_channels, new.out_channels, new.supported_modalities) == before + assert before == (old.in_channels, old.out_channels, old.supported_modalities) + assert new.supported_modalities == {'vision', 'text'} + + +@pytest.mark.parametrize('columns', [ + ['image_file_name'], ['image_path'], ['filename'], ['sentence'], ['text'], + ['video_path'], ['audio_path'], ['mystery_col'], ['image_file_name', 'sentence'], +]) +@pytest.mark.parametrize('form', ['extractor', 'activations_model_mapping', + 'shared_activations_model']) +def test_detect_modalities_matches_private_alias(form, columns): + model = _forms()[form][0] + stimuli = _Columns(columns) + assert model.detect_modalities(stimuli) == model._detect_modalities(stimuli) + assert model.detect_modalities(stimuli) == model._dispatcher.detect_modalities(stimuli) + + +def test_detect_modalities_private_alias_is_same_function(): + assert BrainScoreModel._detect_modalities is BrainScoreModel.detect_modalities + + +def test_extractor_mapping_survives_reset_and_reuse(): + vision = _FakeExtractor('v-out') + model = BrainScoreModel('m', extractor={'vision': vision}) + mapping = model.extractor + stimuli = _Columns(['image_path']) + assert model.process(stimuli) == 'v-out' + model.reset() + assert vision.resets == 1 + assert model.extractor['vision'] is vision and dict(mapping) == {'vision': vision} + assert model.process(stimuli) == 'v-out' + assert [c[0] for c in vision.calls] == [stimuli, stimuli] + + +def test_extractor_access_imports_no_framework(): + code = ( + "import sys\n" + "from brainscore_core.model_interface import BrainScoreModel\n" + "m = BrainScoreModel('m', extractor={'text': lambda stimuli, layers: None})\n" + "dict(m.extractor); m.detect_modalities(type('S', (), {'columns': ['sentence']})())\n" + "bad = [n for n in ('torch', 'transformers', 'tensorflow', 'jax') if n in sys.modules]\n" + "assert not bad, bad\n" + ) + subprocess.run([sys.executable, '-c', code], check=True) diff --git a/tests/test_native_subject.py b/tests/test_native_subject.py index 2ef1c313..a6b09a7d 100644 --- a/tests/test_native_subject.py +++ b/tests/test_native_subject.py @@ -22,7 +22,7 @@ class TextSubject(Subject): def __init__(self): self.seen = [] - def interact(self, session): + def participate(self, session): while (event := session.next_input()) is not None: self.seen.append(event) if event.channel != "text": @@ -59,7 +59,7 @@ def test_minimal_subject_consumes_and_emits_timestamped_events(): "supported_modalities", "check_session_support"): assert not hasattr(subject, name) session = InMemorySession([StreamEvent("text", "cat", 12.0, {"stimulus_id": "a"})]) - subject.interact(session) + subject.participate(session) assert [event.channel for event in session.emitted] == [ "neural:language_network", "behavior", ] @@ -85,14 +85,14 @@ def emit(self, event): self.responses.append(event.payload) session = FeedbackSession() - TextSubject().interact(session) + TextSubject().participate(session) assert session.responses == ["CAT", "CAT"] -@pytest.mark.parametrize("missing", ["identifier", "in_channels", "out_channels", "interact"]) +@pytest.mark.parametrize("missing", ["identifier", "in_channels", "out_channels", "participate"]) def test_incomplete_native_subject_is_rejected(missing): declarations = dict(identifier="incomplete", in_channels={"text"}, - out_channels={"behavior"}, interact=lambda self, session: None) + out_channels={"behavior"}, participate=lambda self, session: None) del declarations[missing] incomplete = type("Incomplete", (Subject,), declarations) with pytest.raises(TypeError, match=missing): @@ -105,7 +105,7 @@ class Stateless(Subject): in_channels = {"text"} out_channels = {"behavior"} - def interact(self, session): + def participate(self, session): pass first, second = Stateless(), Stateless() diff --git a/tests/test_participate_contract.py b/tests/test_participate_contract.py new file mode 100644 index 00000000..9a97dae8 --- /dev/null +++ b/tests/test_participate_contract.py @@ -0,0 +1,213 @@ +"""Contract conformance for ``participate(session)`` (Contract Rules v3, rule 1). + +A subject that implements only ``participate`` must run through every public +path: the static typed helpers, the legacy-benchmark facade and a closed-loop +environment. An added capability that claims the same outputs as a built-in +part runs instead of it. No class carries a second evaluation method. +""" +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from brainscore_core import capabilities +from brainscore_core.capabilities import Capability, register_capability +from brainscore_core.compatibility import check_channel_compatibility +from brainscore_core.contract import Subject, TaskContext, UnifiedModel +from brainscore_core.environment import DiscreteSpec, MappingSpec, TextSpec +from brainscore_core.events import EnvironmentStep +from brainscore_core.legacy_facade import LegacyBenchmarkFacade +from brainscore_core.model_interface import BrainScoreModel +from brainscore_core.streaming import BufferedSession, StreamEvent +from brainscore_core.streaming_helpers import ( + behavioral_response, neural_response, run_environment, stimulus_session, +) +from brainscore_core.supported_data_standards.brainio.assemblies import ( + BehavioralAssembly, NeuroidAssembly, +) +from brainscore_core.supported_data_standards.brainio.stimuli import StimulusSet + + +def _images(n=2): + stimuli = StimulusSet(pd.DataFrame({ + 'stimulus_id': [f's{i}' for i in range(n)], + 'image_path': [f'/tmp/s{i}.png' for i in range(n)], + })) + stimuli.identifier = f'participate-images-{n}' + return stimuli + + +def _neural(n): + return NeuroidAssembly(np.arange(n, dtype=float)[:, None], + coords={'stimulus_id': ('presentation', [f's{i}' for i in range(n)]), + 'neuroid_id': ('neuroid', [0])}, + dims=['presentation', 'neuroid']) + + +class _ParticipateOnly(Subject): + """Implements only the contract: no process, look_at or start_recording.""" + + identifier = 'participate-only' + in_channels = {'vision', 'observation'} + out_channels = {'neural:IT', 'behavior', 'motor'} + + def __init__(self): + self.inputs = [] + self.payloads = [] + + def participate(self, session): + if 'motor' in session.requested_output_channels: + # Closed loop: answer each observation until the episode says it ended. + while (step := session.next_input()) is not None and not step.is_last: + self.inputs.append(step) + session.respond(StreamEvent('motor', len(self.inputs) % 3, None)) + return + while (event := session.next_input()) is not None: + self.inputs.append(event) + n = len(session.stimulus_set) + for channel in session.requested_output_channels: + payload = (_neural(n) if channel.startswith('neural:') else + BehavioralAssembly(np.zeros((n, 1)), dims=['presentation', 'choice'])) + self.payloads.append(payload) + session.respond(StreamEvent(channel, payload, None)) + + +def test_subject_has_exactly_one_abstract_evaluation_method(): + assert Subject.__abstractmethods__ == { + 'identifier', 'in_channels', 'out_channels', 'participate'} + assert not hasattr(Subject, 'process') + # Typed bases add conveniences, never a second abstract evaluation method. + assert {'participate', 'process'}.isdisjoint(UnifiedModel.__abstractmethods__) + for cls in (Subject, UnifiedModel, BrainScoreModel, Capability, LegacyBenchmarkFacade): + assert not hasattr(cls, 'interact'), cls + + +def test_static_neural_helper_runs_a_participate_only_subject(): + subject = _ParticipateOnly() + out = neural_response(subject, _images(3), record='IT') + assert out is subject.payloads[0] + assert len(subject.inputs) == 3 + + +def test_static_behavior_helper_runs_a_participate_only_subject(): + subject = _ParticipateOnly() + context = TaskContext(task_type='probabilities', label_set=['a', 'b'], + metadata={'stimulus_set': _images(2)}) + out = behavioral_response(subject, context) + assert out is subject.payloads[0] + + +def test_legacy_facade_runs_a_participate_only_subject(): + subject = _ParticipateOnly() + facade = LegacyBenchmarkFacade(subject, visual_degrees=8) + facade.start_recording('IT', time_bins=[(70, 170)]) + out = facade.look_at(_images(2), number_of_trials=1) + np.testing.assert_array_equal(out.values, [[0.], [1.]]) + assert not hasattr(subject, 'look_at') and not hasattr(subject, 'process') + + +class _Environment: + """Three observations, then a terminal one.""" + + def __init__(self): + self.actions = [] + self.closed = 0 + + def observation_spec(self): + return MappingSpec({'task': TextSpec()}) + + def action_spec(self): + return DiscreteSpec(3) + + def _step(self): + last = len(self.actions) == 3 + return EnvironmentStep(observation={'task': 'move'}, step_num=len(self.actions), + is_last=last, is_terminal=last) + + def reset(self, *, seed=None, options=None): + return self._step() + + def step(self, action): + self.actions.append(action) + return self._step() + + def close(self): + self.closed += 1 + + +def test_closed_loop_runs_a_participate_only_subject(): + subject, env = _ParticipateOnly(), _Environment() + closed_loop = SimpleNamespace(identifier='closed-loop', uses_session=True, + required_input_channels={'observation'}, + requested_output_channels={'motor'}) + match = check_channel_compatibility(subject, closed_loop) + assert match.consumed_inputs == {'observation'} and match.kind == 'full' + applied = run_environment(subject, env) + assert env.actions == [1, 2, 0] and env.closed == 1 + assert applied == [1, 2, 0] + + +@pytest.fixture +def registry(monkeypatch): + saved = dict(capabilities.capability_registry) + yield + capabilities.capability_registry.clear() + capabilities.capability_registry.update(saved) + + +def test_added_capability_takes_precedence_over_the_built_in_part(registry): + calls = [] + + class Custom(Capability): + identifier = 'participate-precedence-test' + output_channels = {'neural:IT'} + + def handles(self, model, event, **kwargs): + return False + + def process(self, model, event, **kwargs): + raise NotImplementedError + + def supports_session(self, model, channels): + return set(channels) == {'neural:IT'} + + def participate(self, model, session): + while session.next_input() is not None: + pass + calls.append(model.identifier) + session.respond(StreamEvent('neural:IT', _neural(len(session.stimulus_set)), None)) + + def extractor(stimuli, layers=None, **kwargs): + pytest.fail('built-in neural part ran instead of the added capability') + + register_capability(Custom()) + model = BrainScoreModel('precedence', region_layer_map={'IT': 'layer'}, + extractor={'vision': extractor}) + assert model.check_session_support({'neural:IT'}).identifier == Custom.identifier + out = neural_response(model, _images(2), record='IT') + assert calls == ['precedence'] and out.sizes['presentation'] == 2 + + +def _features(stimuli, layers=None, **kwargs): + return _neural(len(stimuli)).assign_coords(layer=('neuroid', [layers[0]])) + + +@pytest.mark.parametrize('streaming, meta', [ + (False, [{'driver': 'interact', 'record': 'IT'}]), + (True, [{'driver': 'interact_streaming', 'record': 'IT', 'stream_index': i, 'streaming': True} + for i in range(2)]), +]) +def test_typed_drivers_keep_the_recorded_pre_rename_driver_labels(streaming, meta): + """Response meta written before the rename must still compare equal. + + ``driver`` is a record-format label, not the method name; saved records + compare meta exactly (``compare_outputs``), so the label is frozen. + """ + model = BrainScoreModel('labels', region_layer_map={'IT': 'layer'}, + extractor={'vision': _features}) + session = (BufferedSession(stimulus_set=_images(2), requested_output_channels=['neural:IT'], + streaming=True) + if streaming else stimulus_session(_images(2), record='IT')) + model.participate(session) + assert [event.meta for event in session.emitted] == meta diff --git a/tests/test_session_view.py b/tests/test_session_view.py index 76618ce5..91128e1c 100644 --- a/tests/test_session_view.py +++ b/tests/test_session_view.py @@ -70,7 +70,7 @@ def test_undeclared_events_are_skipped_and_declared_ones_delivered(): StreamEvent('vision', '/tmp/a.png', 0.0, meta={'stimulus_id': 'a', 'column': 'image_path'}), ], requested_output_channels=['neural:IT']) - model.interact(session) + model.participate(session) seen, = extractor.seen assert list(seen['stimulus_id']) == ['a'] assert session.collect('neural:IT').sizes['presentation'] == 1 @@ -81,7 +81,7 @@ def test_environment_steps_pass_through_to_the_policy(): model = BrainScoreModel('policy', policy=lambda step: steps.append(step) or EnvironmentResponse(action=1)) step = EnvironmentStep(observation={'value': 1}) session = BufferedSession([step], requested_output_channels=['motor']) - model.interact(session) + model.participate(session) assert steps == [step] assert session.emitted[0].payload.action == 1 diff --git a/tests/test_streaming_behavior.py b/tests/test_streaming_behavior.py index c978731e..912457fa 100644 --- a/tests/test_streaming_behavior.py +++ b/tests/test_streaming_behavior.py @@ -31,7 +31,7 @@ def _stimuli(n=5): class _Session: - """Minimal behavioral session; `streaming` decides which driver interact picks.""" + """Minimal behavioral session; `streaming` decides which driver participate picks.""" def __init__(self, task_context, streaming=False): self.task_context = task_context diff --git a/tests/test_streaming_helpers.py b/tests/test_streaming_helpers.py index 4eabc92d..f961dfe5 100644 --- a/tests/test_streaming_helpers.py +++ b/tests/test_streaming_helpers.py @@ -128,18 +128,18 @@ def __call__(self, stimuli, layers=None, **kwargs): ) -class _InteractTrackingBrainScoreModel(BrainScoreModel): +class _ParticipateTrackingBrainScoreModel(BrainScoreModel): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.interact_called = False - self.interact_requested_output_channels = None + self.participate_called = False + self.participate_requested_output_channels = None - def interact(self, session): - self.interact_called = True - self.interact_requested_output_channels = tuple( + def participate(self, session): + self.participate_called = True + self.participate_requested_output_channels = tuple( session.requested_output_channels ) - return super().interact(session) + return super().participate(session) def _native_neural_model(extractor, model_cls=BrainScoreModel): @@ -257,15 +257,15 @@ def process(self, input_event, multi_modality=False): class _TrackingTemporalBrainScoreModel(_TemporalBrainScoreModel): def __init__(self): super().__init__() - self.interact_called = False - self.interact_requested_output_channels = None + self.participate_called = False + self.participate_requested_output_channels = None - def interact(self, session): - self.interact_called = True - self.interact_requested_output_channels = tuple( + def participate(self, session): + self.participate_called = True + self.participate_requested_output_channels = tuple( session.requested_output_channels ) - return super().interact(session) + return super().participate(session) class _SyntheticSubject(UnifiedModel): @@ -401,7 +401,7 @@ def test_neural_response_matches_process_assembly_exactly(): subject = _SyntheticSubject(expected) stimuli = _stimulus_set() - assert type(subject).interact is UnifiedModel.interact + assert type(subject).participate is UnifiedModel.participate scored = neural_response(subject, stimuli, record="IT") assert subject.recording_target == "IT" @@ -409,7 +409,7 @@ def test_neural_response_matches_process_assembly_exactly(): xr.testing.assert_identical(scored, expected) -def test_neural_response_uses_native_interact_and_preserves_metadata_exactly(): +def test_neural_response_uses_native_participate_and_preserves_metadata_exactly(): stimuli = _metadata_stimulus_set() legacy_extractor = _MetadataPreservingExtractor() @@ -419,18 +419,18 @@ def test_neural_response_uses_native_interact_and_preserves_metadata_exactly(): native_extractor = _MetadataPreservingExtractor() subject = _native_neural_model( - native_extractor, model_cls=_InteractTrackingBrainScoreModel + native_extractor, model_cls=_ParticipateTrackingBrainScoreModel ) scored = neural_response(subject, stimuli, record="IT") - assert subject.interact_called is True - assert subject.interact_requested_output_channels == ("neural:IT",) + assert subject.participate_called is True + assert subject.participate_requested_output_channels == ("neural:IT",) assert native_extractor.last_stimuli is stimuli assert "modality" not in scored.coords xr.testing.assert_identical(scored, expected) -def test_neural_response_native_interact_matches_multimodal_process_exactly(): +def test_neural_response_native_participate_matches_multimodal_process_exactly(): stimuli = _multimodal_stimulus_set() legacy, _, _ = _native_multimodal_model() @@ -438,19 +438,19 @@ def test_neural_response_native_interact_matches_multimodal_process_exactly(): expected = legacy.process(stimuli, multi_modality=True) subject, activations, text = _native_multimodal_model( - model_cls=_InteractTrackingBrainScoreModel + model_cls=_ParticipateTrackingBrainScoreModel ) scored = neural_response(subject, stimuli, record="IT") - assert subject.interact_called is True - assert subject.interact_requested_output_channels == ("neural:IT",) + assert subject.participate_called is True + assert subject.participate_requested_output_channels == ("neural:IT",) assert activations.call_args is not None assert text.call_args is not None assert set(scored["modality"].values.tolist()) == {"vision", "text"} xr.testing.assert_identical(scored, expected) -def test_neural_response_native_interact_matches_cross_tower_process_exactly(): +def test_neural_response_native_participate_matches_cross_tower_process_exactly(): stimuli = _multimodal_stimulus_set() legacy, _, _ = _native_cross_tower_model() @@ -458,19 +458,19 @@ def test_neural_response_native_interact_matches_cross_tower_process_exactly(): expected = legacy.process(stimuli, multi_modality=True) subject, activations, text = _native_cross_tower_model( - model_cls=_InteractTrackingBrainScoreModel + model_cls=_ParticipateTrackingBrainScoreModel ) scored = neural_response(subject, stimuli, record="all") - assert subject.interact_called is True - assert subject.interact_requested_output_channels == ("neural:all",) + assert subject.participate_called is True + assert subject.participate_requested_output_channels == ("neural:all",) assert activations.call_args["layers"] == ["vision_model.layer.10"] assert text.call_args["layers"] == ["text_model.layer.20"] assert set(scored["modality"].values.tolist()) == {"vision", "text"} xr.testing.assert_identical(scored, expected) -def test_neural_response_native_interact_threads_time_bins_exactly(): +def test_neural_response_native_participate_threads_time_bins_exactly(): stimuli = _metadata_stimulus_set() time_bins = [(70, 170), (170, 270)] @@ -481,8 +481,8 @@ def test_neural_response_native_interact_threads_time_bins_exactly(): subject = _TrackingTemporalBrainScoreModel() scored = neural_response(subject, stimuli, record="IT", time_bins=time_bins) - assert subject.interact_called is True - assert subject.interact_requested_output_channels == ("neural:IT",) + assert subject.participate_called is True + assert subject.participate_requested_output_channels == ("neural:IT",) assert scored.dims == ("presentation", "time_bin", "neuroid") assert list(scored["time_bin_start_ms"].values) == [70, 170] assert list(scored["time_bin_end_ms"].values) == [170, 270] @@ -499,8 +499,8 @@ def test_neural_response_without_time_bins_keeps_2d_output(): subject = _TrackingTemporalBrainScoreModel() scored = neural_response(subject, stimuli, record="IT") - assert subject.interact_called is True - assert subject.interact_requested_output_channels == ("neural:IT",) + assert subject.participate_called is True + assert subject.participate_requested_output_channels == ("neural:IT",) assert scored.dims == ("presentation", "neuroid") assert "time_bin" not in scored.dims xr.testing.assert_identical(scored, expected) @@ -625,13 +625,13 @@ def test_behavioral_response_matches_legacy_start_task_process_exactly(): expected = legacy.process(scoring) helper_subject = _behavior_model( - features, model_cls=_InteractTrackingBrainScoreModel + features, model_cls=_ParticipateTrackingBrainScoreModel ) helper_context = _behavior_context(fitting, scoring) scored = behavioral_response(helper_subject, helper_context) - assert helper_subject.interact_called is True - assert helper_subject.interact_requested_output_channels == ("behavior",) + assert helper_subject.participate_called is True + assert helper_subject.participate_requested_output_channels == ("behavior",) assert isinstance(scored, BehavioralAssembly) xr.testing.assert_identical(scored, expected) @@ -672,13 +672,13 @@ def test_behavioral_response_matches_legacy_generation_path_exactly(): helper_calls = [] helper_subject = _generation_model( - helper_calls, model_cls=_InteractTrackingBrainScoreModel + helper_calls, model_cls=_ParticipateTrackingBrainScoreModel ) helper_context = _generation_context(scoring) scored = behavioral_response(helper_subject, helper_context) - assert helper_subject.interact_called is True - assert helper_subject.interact_requested_output_channels == ("behavior",) + assert helper_subject.participate_called is True + assert helper_subject.participate_requested_output_channels == ("behavior",) assert isinstance(scored, BehavioralAssembly) xr.testing.assert_identical(scored, expected) assert helper_calls == legacy_calls @@ -705,7 +705,7 @@ def test_behavioral_response_falls_back_for_non_native_subject(): scored = behavioral_response(subject, context) - assert type(subject).interact is UnifiedModel.interact + assert type(subject).participate is UnifiedModel.participate assert subject.process_input is scoring xr.testing.assert_identical(scored, expected) @@ -826,13 +826,13 @@ def test_environment_session_advances_after_motor_emit(): assert second_step.step_num == 1 -def test_native_environment_interact_preserves_applied_legacy_actions(): +def test_native_environment_participate_preserves_applied_legacy_actions(): session = environment_session(_SyntheticEnvironment(_environment_steps(4))) native_model = _native_environment_model( - _droid_action_fn, model_cls=_InteractTrackingBrainScoreModel + _droid_action_fn, model_cls=_ParticipateTrackingBrainScoreModel ) - native_model.interact(session) + native_model.participate(session) native_trajectory = session.collect("motor") legacy_model = _make_environment_model(action_fn=_droid_action_fn) @@ -840,8 +840,8 @@ def test_native_environment_interact_preserves_applied_legacy_actions(): # The old loop predicted once after termination; that response was never applied. expected_trajectory = _legacy_environment_rollout(legacy_model, legacy_env)[:-1] - assert native_model.interact_called is True - assert native_model.interact_requested_output_channels == ("motor",) + assert native_model.participate_called is True + assert native_model.participate_requested_output_channels == ("motor",) assert [event.channel for event in session.emitted] == [ "motor", "motor", "motor", ] @@ -878,7 +878,7 @@ def test_environment_session_emits_motor_events_for_multistep_rollout(): def test_run_environment_preserves_applied_legacy_actions(): helper_model = _native_environment_model( - _droid_action_fn, model_cls=_InteractTrackingBrainScoreModel + _droid_action_fn, model_cls=_ParticipateTrackingBrainScoreModel ) helper_env = _SyntheticEnvironment(_environment_steps(4)) helper_trajectory = run_environment(helper_model, helper_env) @@ -899,8 +899,8 @@ def test_run_environment_preserves_applied_legacy_actions(): assert len(helper_env.actions) == len(legacy_env.actions) for helper_action, expected_action in zip(helper_env.actions, legacy_env.actions): np.testing.assert_array_equal(helper_action, expected_action) - assert helper_model.interact_called is True - assert helper_model.interact_requested_output_channels == ("motor",) + assert helper_model.participate_called is True + assert helper_model.participate_requested_output_channels == ("motor",) def test_run_environment_falls_back_for_non_native_subjects(): @@ -909,7 +909,7 @@ def test_run_environment_falls_back_for_non_native_subjects(): trajectory = run_environment(subject, env) - assert type(subject).interact is UnifiedModel.interact + assert type(subject).participate is UnifiedModel.participate assert subject.processed_steps == [0, 1] assert [response.metadata["step_num_seen"] for response in trajectory] == [ 0, 1, @@ -918,25 +918,25 @@ def test_run_environment_falls_back_for_non_native_subjects(): def test_run_environment_terminates_cleanly_for_zero_and_one_step_envs(): model = _native_environment_model( - _droid_action_fn, model_cls=_InteractTrackingBrainScoreModel + _droid_action_fn, model_cls=_ParticipateTrackingBrainScoreModel ) zero_env = _SyntheticEnvironment([]) assert run_environment(model, zero_env) == [] assert zero_env.actions == [] - assert model.interact_called is True - assert model.interact_requested_output_channels == ("motor",) + assert model.participate_called is True + assert model.participate_requested_output_channels == ("motor",) model = _native_environment_model( - _droid_action_fn, model_cls=_InteractTrackingBrainScoreModel + _droid_action_fn, model_cls=_ParticipateTrackingBrainScoreModel ) one_env = _SyntheticEnvironment(_environment_steps(1)) trajectory = run_environment(model, one_env) assert trajectory == [] # Already terminal: no action is requested. assert one_env.actions == [] - assert model.interact_called is True - assert model.interact_requested_output_channels == ("motor",) + assert model.participate_called is True + assert model.participate_requested_output_channels == ("motor",) def _ablation_state_change(): @@ -1026,12 +1026,12 @@ def test_apply_state_change_matches_legacy_process_result(): fn_helper, _ = _counting_state_change_fn() helper_model = _native_state_change_model( - fn_helper, model_cls=_InteractTrackingBrainScoreModel + fn_helper, model_cls=_ParticipateTrackingBrainScoreModel ) helper_result = apply_state_change(helper_model, _ablation_state_change()) - assert helper_model.interact_called is True - assert helper_model.interact_requested_output_channels == ("perturbation",) + assert helper_model.participate_called is True + assert helper_model.participate_requested_output_channels == ("perturbation",) assert isinstance(helper_result, PerturbationApplied) assert helper_result.handle_id == legacy_result.handle_id assert helper_result.target == legacy_result.target @@ -1057,20 +1057,20 @@ def cleanup(): return applied, cleanup model = _native_state_change_model( - state_change_fn, model_cls=_InteractTrackingBrainScoreModel + state_change_fn, model_cls=_ParticipateTrackingBrainScoreModel ) applied = apply_state_change(model, _ablation_state_change()) assert model_state["output"] == 0.0 - assert model.interact_requested_output_channels == ("perturbation",) + assert model.participate_requested_output_channels == ("perturbation",) reset = StateChange(kind="reset", handle_id=applied.handle_id) reset_session = state_change_session(reset) reset_event = reset_session.next_input() - model.interact_called = False + model.participate_called = False reset_result = apply_state_change(model, reset) - assert model.interact_called is True - assert model.interact_requested_output_channels == ("perturbation",) + assert model.participate_called is True + assert model.participate_requested_output_channels == ("perturbation",) assert reset_event.channel == "lesion:ablation-handle" assert reset_event.meta["reset"] == applied.handle_id assert reset_result == applied.handle_id @@ -1088,7 +1088,7 @@ def test_apply_state_change_falls_back_for_non_native_subject(): result = apply_state_change(subject, state_change) - assert type(subject).interact is UnifiedModel.interact + assert type(subject).participate is UnifiedModel.participate assert subject.process_input is state_change assert result is applied @@ -1145,3 +1145,127 @@ def test_neural_response_rejects_several_regions(self): from brainscore_core.streaming_helpers import neural_response with pytest.raises(TypeError, match='single region'): neural_response(object(), self._stimuli(), record=['V4', 'IT']) + + +class _TypedSubject: + """A plain object with the typed methods the public drivers call. + + No base class: the drivers rely on these calls only, never on discovery. + """ + + def __init__(self, output=None): + self.calls = [] + self.output = output + + def start_recording(self, target, time_bins=None): + self.calls.append(('start_recording', target, time_bins)) + + def start_task(self, task_context): + self.calls.append(('start_task', task_context)) + + def process(self, stimuli, **kwargs): + self.calls.append(('process', stimuli, kwargs)) + return self.output + + +class TestPublicSessionHelpers: + """The promoted session helpers keep their behavior under the public names.""" + + def test_private_names_are_the_same_functions(self): + from brainscore_core import streaming_helpers as helpers + assert helpers._drain_stream_events is helpers.drain_session_inputs + assert helpers._stimuli_from_stream_session is helpers.session_stimuli + assert helpers._drive_neural_session_via_process is helpers.drive_neural_session + assert helpers._drive_behavior_session_via_process is helpers.drive_behavior_session + + def test_drain_returns_every_input_in_order_then_none(self): + from brainscore_core.streaming import BufferedSession + from brainscore_core.streaming_helpers import drain_session_inputs + events = [StreamEvent('vision', f'p{index}', float(index), {'i': index}) + for index in range(3)] + session = BufferedSession(events, requested_output_channels=('neural:IT',)) + assert drain_session_inputs(session) == events + assert session.next_input() is None + assert drain_session_inputs(session) == [] + + def test_drain_rejects_non_events(self): + from brainscore_core.streaming_helpers import drain_session_inputs + + class _Session: + def next_input(self): + return 'not an event' + with pytest.raises(TypeError, match='StreamEvent'): + drain_session_inputs(_Session()) + + def test_session_stimuli_returns_the_session_table_itself(self): + from brainscore_core.streaming_helpers import ( + drain_session_inputs, session_stimuli) + stimuli = _stimulus_set() + session = stimulus_session(stimuli, record='IT') + assert session_stimuli(session, drain_session_inputs(session)) is stimuli + + def test_session_stimuli_rebuilds_inputs_without_a_table(self): + from brainscore_core.streaming_helpers import session_stimuli + events = [StreamEvent('vision', 'a.png', 0.0, + {'stimulus_id': 's0', 'column': 'image_path'})] + + class _Session: + pass + rebuilt = session_stimuli(_Session(), events) + assert list(rebuilt['stimulus_id']) == ['s0'] + assert list(rebuilt['image_path']) == ['a.png'] + + def test_drive_neural_session_one_process_call_with_time_bins(self): + from brainscore_core.streaming_helpers import drive_neural_session + stimuli = _stimulus_set() + output = _assembly() + subject = _TypedSubject(output) + session = stimulus_session(stimuli, record='IT', time_bins=[(70, 170)]) + drive_neural_session(subject, session) + assert subject.calls == [('start_recording', 'IT', [(70, 170)]), + ('process', stimuli, {})] + event, = session.emitted + assert event.channel == 'neural:IT' + assert event.payload is output + assert event.meta == {'driver': 'interact', 'record': 'IT'} + + def test_drive_neural_session_refuses_variance_before_compute(self): + from brainscore_core.streaming_helpers import drive_neural_session + subject = _TypedSubject() + session = stimulus_session(_stimulus_set(), record='IT') + session.settings = {'require_variance': True, 'number_of_trials': 3} + with pytest.raises(NotImplementedError, match='number_of_trials=3'): + drive_neural_session(subject, session) + assert subject.calls == [] + assert session.emitted == [] + + def test_drive_behavior_session_starts_task_once_and_scores_the_task_table(self): + from brainscore_core.streaming_helpers import drive_behavior_session + scored = _stimulus_set() + context = TaskContext(task_type='probabilities', label_set=['a', 'b'], + fitting_stimuli=_stimulus_set(), + metadata={'stimulus_set': scored}) + output = BehavioralAssembly( + np.array([[0.9, 0.1], [0.2, 0.8]]), + coords={'stimulus_id': ('presentation', ['s0', 's1']), + 'choice': ('choice', ['a', 'b'])}, + dims=['presentation', 'choice']) + subject = _TypedSubject(output) + session = behavior_session(context) + drive_behavior_session(subject, session) + assert subject.calls == [('start_task', context), ('process', scored, {})] + event, = session.emitted + assert event.channel == 'behavior' + assert event.payload is output + assert event.meta == {'driver': 'interact', 'task_type': 'probabilities'} + + def test_drive_behavior_session_refuses_variance_before_compute(self): + from brainscore_core.streaming_helpers import drive_behavior_session + context = TaskContext(task_type='probabilities', + metadata={'stimulus_set': _stimulus_set()}) + subject = _TypedSubject() + session = behavior_session(context) + session.settings = {'require_variance': True} + with pytest.raises(NotImplementedError, match='trial-to-trial variance'): + drive_behavior_session(subject, session) + assert subject.calls == [] diff --git a/tests/test_subject_channels.py b/tests/test_subject_channels.py index 29302175..03869497 100644 --- a/tests/test_subject_channels.py +++ b/tests/test_subject_channels.py @@ -58,15 +58,15 @@ def required_channels(self): assert subject.required_channels == {"instruction"} -def test_typed_interact_default_runs_sessions_and_rejects_unknown_outputs(): +def test_typed_participate_default_runs_sessions_and_rejects_unknown_outputs(): from brainscore_core.streaming import BufferedSession subject = _ChannelSubject() with pytest.raises(NotImplementedError, match="runs sessions requesting"): - subject.interact(BufferedSession(requested_output_channels=["pupil"])) + subject.participate(BufferedSession(requested_output_channels=["pupil"])) -def test_legacy_subclass_still_instantiates_without_interact(): +def test_legacy_subclass_still_instantiates_without_participate(): class ExistingSubject(UnifiedModel): @property def identifier(self): diff --git a/tests/test_subject_rename.py b/tests/test_subject_rename.py index 5ad0a01a..aeac705d 100644 --- a/tests/test_subject_rename.py +++ b/tests/test_subject_rename.py @@ -12,7 +12,7 @@ class _ConcreteSubject(Subject): in_channels = {"vision"} out_channels = {"behavior"} - def interact(self, session): + def participate(self, session): pass diff --git a/tests/test_typed_convenience.py b/tests/test_typed_convenience.py index 144ce5c9..208825ef 100644 --- a/tests/test_typed_convenience.py +++ b/tests/test_typed_convenience.py @@ -27,4 +27,4 @@ def test_typed_subclass_instantiates_without_process_and_fails_clearly(): stimuli.identifier = 'one' session = BufferedSession(stimulus_set=stimuli, requested_output_channels=['neural:IT']) with pytest.raises(NotImplementedError, match='defines no process'): - subject.interact(session) + subject.participate(session)