From a5448cb938107875c7d9fd0304fbb4c33eb4a8c1 Mon Sep 17 00:00:00 2001 From: Kartik Pradeepan Date: Sun, 1 Nov 2026 11:23:13 -0500 Subject: [PATCH 1/9] feat(score): stamp the compatibility match on every score Pre-flight is one check_compatibility call; its match (full or partial, consumed and dropped inputs) is recorded in score.attrs['compatibility']. --- brainscore_language/__init__.py | 5 ++--- tests/test_preflight.py | 3 ++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/brainscore_language/__init__.py b/brainscore_language/__init__.py index 9052ab6b..33c569f8 100644 --- a/brainscore_language/__init__.py +++ b/brainscore_language/__init__.py @@ -60,7 +60,6 @@ def _run_score(model_identifier: str, benchmark_identifier: str, Score the model referenced by the `model_identifier` on the benchmark referenced by the `benchmark_identifier`. """ from brainscore_core.compatibility import ( - check_channel_compatibility, check_compatibility, ) from brainscore_core.extraction_cache import weight_fingerprint_scope @@ -78,8 +77,7 @@ def _run_score(model_identifier: str, benchmark_identifier: str, requested_channels = requested_output_channels_for_score(benchmark) # Pre-flight checks - check_compatibility(model, benchmark) - check_channel_compatibility(model, benchmark) + compatibility = check_compatibility(model, benchmark) if check_mem: check_memory(model, benchmark) @@ -97,6 +95,7 @@ def _run_score(model_identifier: str, benchmark_identifier: str, protocol=infer_score_protocol(model, benchmark, requested_channels), harness_id='brainscore_language', ) + score.attrs['compatibility'] = compatibility.as_attrs() return score diff --git a/tests/test_preflight.py b/tests/test_preflight.py index 3d013353..8058cdd7 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -45,7 +45,7 @@ def test_incompatible_model_raises_compatibility_error( mock_load_model.return_value = model mock_load_benchmark.return_value = benchmark - with pytest.raises(CompatibilityError, match="does not support modalities required"): + with pytest.raises(CompatibilityError, match="takes in none of the inputs"): _run_score('test-model', 'test-bench', check_mem=False) @patch('brainscore_language.load_benchmark') @@ -95,6 +95,7 @@ def test_score_metadata_attrs_are_stamped( assert score.attrs['out_channels'] == ('behavior',) assert score.attrs['protocol'] == 'behavior:readout' assert score.attrs['subject_id'] == 'test-model' + assert score.attrs['compatibility']['match'] == 'full' assert score.attrs['harness_id'] == 'brainscore_language' @patch('brainscore_core.compatibility.check_channel_compatibility') From c144b9f224cea3cb1717ed5d7c5ebff02e5a18d3 Mon Sep 17 00:00:00 2001 From: Kartik Pradeepan Date: Sun, 1 Nov 2026 11:23:13 -0500 Subject: [PATCH 2/9] refactor(compat): stop forwarding unknown attributes to the legacy model The adapter exposes the wrapped model as legacy_model instead of passing every unknown attribute through, so benchmarks see only what the adapter declares. --- brainscore_language/compat/unified_adapter.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/brainscore_language/compat/unified_adapter.py b/brainscore_language/compat/unified_adapter.py index 2245dc34..913ebdc0 100644 --- a/brainscore_language/compat/unified_adapter.py +++ b/brainscore_language/compat/unified_adapter.py @@ -178,6 +178,11 @@ def reset(self) -> None: if hasattr(self._legacy, 'current_tokens'): self._legacy.current_tokens = None - def __getattr__(self, name): - # Delegate attribute access to the legacy model for backwards compatibility - return getattr(self._legacy, name) + @property + def legacy_model(self): + """The wrapped legacy model, for tools that need its own attributes. + + Unknown attributes are no longer forwarded implicitly: a benchmark + sees only what the adapter declares (Contract Rules v3, rule 2.1). + """ + return self._legacy From eb4b488ba3efd535a16cb2476483dc98f7cc0de0 Mon Sep 17 00:00:00 2001 From: Kartik Pradeepan Date: Sun, 1 Nov 2026 16:51:20 -0500 Subject: [PATCH 3/9] feat(score): run legacy benchmarks through the facade with one joint digest_text load_benchmark marks each benchmark's calling convention and _run_score hands a legacy benchmark the permanent LegacyBenchmarkFacade. The Pereira -unified variant declares the typed convention and is not wrapped. LanguageModelAdapter declares its behavior output and, for a facade session, configures the legacy model as the benchmark did and makes the one digest_text call the benchmark would have made, returning neural and behavioral outputs together. Other sessions keep the typed drivers. --- brainscore_language/__init__.py | 10 +++- .../benchmarks/pereira2018/unified.py | 4 ++ brainscore_language/compat/unified_adapter.py | 44 +++++++++++++- tests/test_legacy_score.py | 3 +- tests/test_unified_adapter.py | 58 +++++++++++++++++++ 5 files changed, 114 insertions(+), 5 deletions(-) diff --git a/brainscore_language/__init__.py b/brainscore_language/__init__.py index 33c569f8..22e1357d 100644 --- a/brainscore_language/__init__.py +++ b/brainscore_language/__init__.py @@ -37,8 +37,9 @@ def load_benchmark(identifier: str) -> Benchmark: import_plugin('brainscore_language', 'benchmarks', identifier) from brainscore_core.compatibility import ensure_legacy_benchmark_modalities + from brainscore_core.legacy_facade import mark_legacy_benchmark benchmark = benchmark_registry[identifier]() - return ensure_legacy_benchmark_modalities(benchmark, {'text'}) + return mark_legacy_benchmark(ensure_legacy_benchmark_modalities(benchmark, {'text'})) def load_model(identifier: str) -> 'Subject': @@ -81,10 +82,15 @@ def _run_score(model_identifier: str, benchmark_identifier: str, if check_mem: check_memory(model, benchmark) + # A legacy benchmark calls digest_text and friends; it receives the permanent + # facade, which runs those calls as sessions through the subject's interact. + from brainscore_core.legacy_facade import subject_for_benchmark + candidate = subject_for_benchmark(model, benchmark) + import time as _time _t0 = _time.time() with weight_fingerprint_scope(): - score: Score = benchmark(model) + score: Score = benchmark(candidate) score.attrs['runtime_sec'] = round(_time.time() - _t0, 2) score.attrs['model_identifier'] = model_identifier score.attrs['benchmark_identifier'] = benchmark_identifier diff --git a/brainscore_language/benchmarks/pereira2018/unified.py b/brainscore_language/benchmarks/pereira2018/unified.py index 10e25189..3de1983d 100644 --- a/brainscore_language/benchmarks/pereira2018/unified.py +++ b/brainscore_language/benchmarks/pereira2018/unified.py @@ -31,6 +31,10 @@ class _Pereira2018ExperimentUnified(_Pereira2018Experiment): NeuroidAssembly directly. """ + # Calls the typed convention on the candidate itself; scoring must not + # wrap the candidate in the legacy-benchmark facade. + subject_api = 'typed' + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._identifier = self.identifier + '-unified' diff --git a/brainscore_language/compat/unified_adapter.py b/brainscore_language/compat/unified_adapter.py index 913ebdc0..df9977a9 100644 --- a/brainscore_language/compat/unified_adapter.py +++ b/brainscore_language/compat/unified_adapter.py @@ -11,7 +11,12 @@ from typing import Any, Dict, Optional, Set from brainscore_core.model_interface import UnifiedModel, TaskContext -from brainscore_core.streaming_helpers import _drive_neural_session_via_process +from brainscore_core.streaming import StreamEvent +from brainscore_core.streaming_helpers import ( + _drain_stream_events, + _drive_behavior_session_via_process, + _drive_neural_session_via_process, +) class LanguageModelAdapter(UnifiedModel): @@ -125,8 +130,43 @@ def _process_table(self, stimuli): return xr.concat(outputs, dim='presentation').isel( presentation=np.argsort(positions_out)) + @property + def out_channels(self) -> Set[str]: + # A legacy ArtificialSubject answers behavioral tasks as well as recording. + return super().out_channels | {'behavior'} + def interact(self, session) -> None: - _drive_neural_session_via_process(self, session) + """Run a session; a legacy-benchmark session replays the exact ``digest_text``. + + A session carrying ``legacy_text`` (from the legacy-benchmark facade) + configures the legacy model as the benchmark did and makes one + ``digest_text`` call that can return neural and behavioral outputs + together. Other sessions use the typed drivers over ``process``. + """ + channels = tuple(session.requested_output_channels or ()) + missing = set(channels) - self.out_channels + if not channels or missing: + raise ValueError(f'Undeclared output channels {sorted(missing)}; declared: ' + f'{sorted(self.out_channels)}') + context = session.task_context + settings = dict(getattr(context, 'metadata', None) or {}) + if 'legacy_text' not in settings: + if channels == ('behavior',): + _drive_behavior_session_via_process(self, session) + else: + _drive_neural_session_via_process(self, session) + return + if context.task_type != 'passive': + self.start_task(context) + for channel in channels: + if channel.startswith('neural:'): + self.start_neural_recording(channel.split(':', 1)[1], + settings.get('recording_type', 'fMRI')) + _drain_stream_events(session) + result = self._legacy.digest_text(settings['legacy_text']) + for channel in channels: + key = 'neural' if channel.startswith('neural:') else 'behavior' + session.respond(StreamEvent(channel, result[key], 0.0, meta={'driver': 'legacy'})) # Backwards-compatible: existing benchmarks call digest_text() directly def digest_text(self, text): diff --git a/tests/test_legacy_score.py b/tests/test_legacy_score.py index ea952d5b..ddaffe3e 100644 --- a/tests/test_legacy_score.py +++ b/tests/test_legacy_score.py @@ -13,7 +13,8 @@ class LegacyBenchmark: parent = 'behavior' def __call__(self, model): - assert model.required_modalities == {'text'} + # The benchmark receives the legacy facade; the subject is behind it. + assert model.subject.required_channels == {"text"} return Score(0.5) diff --git a/tests/test_unified_adapter.py b/tests/test_unified_adapter.py index e216f0fc..a17eb6ba 100644 --- a/tests/test_unified_adapter.py +++ b/tests/test_unified_adapter.py @@ -405,3 +405,61 @@ def test_load_model_does_not_double_wrap_unified(self): assert model is native assert not isinstance(model, LanguageModelAdapter) + + +# --- legacy benchmarks through the permanent facade ----------------------- + +@pytest.mark.parametrize('neural,behavior', [(True, False), (False, True), (True, True)]) +def test_legacy_facade_preserves_language_outputs_in_one_call(neural, behavior): + from brainscore_core.legacy_facade import LegacyBenchmarkFacade + from brainscore_core.supported_data_standards.brainio.assemblies import BehavioralAssembly + legacy = _make_legacy_model(region_layer_mapping={'language_system': 'language'}) + expected = {} + if neural: + expected['neural'] = _language_neural_assembly() + if behavior: + expected['behavior'] = BehavioralAssembly(np.array([1., 2.]), dims=['presentation']) + legacy.digest_text.return_value = expected + facade = LegacyBenchmarkFacade(LanguageModelAdapter(legacy)) + if neural: + facade.start_neural_recording('language_system', recording_type='spikes') + if behavior: + facade.start_behavioral_task('reading_times') + text = np.array(['the quick brown', 'fox jumps']) + actual = facade.digest_text(text) + assert actual.keys() == expected.keys() + for key in expected: + assert actual[key] is expected[key] + assert legacy.digest_text.call_count == 1 + assert legacy.digest_text.call_args.args[0] is text + if neural: + legacy.start_neural_recording.assert_called_once_with('language_system', 'spikes') + if behavior: + legacy.start_behavioral_task.assert_called_once_with('reading_times') + + +def test_scoring_wraps_the_adapter_for_a_legacy_language_benchmark(monkeypatch): + import brainscore_language + from brainscore_core.metrics import Score + + class LegacyBenchmark: + identifier = 'legacy-language' + + def __call__(self, candidate): + candidate.start_neural_recording('language_system', 'fMRI') + output = candidate.digest_text(['one sentence', 'another']) + score = Score([1.0], coords={'aggregation': ['center']}, dims=['aggregation']) + score.attrs.update(candidate=type(candidate).__name__, output=output) + return score + + legacy = _make_legacy_model(region_layer_mapping={'language_system': 'language'}) + legacy.digest_text.return_value = {'neural': _language_neural_assembly()} + monkeypatch.setattr(brainscore_language, 'import_plugin', lambda *a, **k: None) + monkeypatch.setitem(brainscore_language.benchmark_registry, 'legacy-language', LegacyBenchmark) + monkeypatch.setattr(brainscore_language, 'load_model', lambda _: LanguageModelAdapter(legacy)) + score = brainscore_language._run_score('m', 'legacy-language', check_mem=False) + assert score.attrs['candidate'] == 'LegacyBenchmarkFacade' + assert score.attrs['output']['neural'] is legacy.digest_text.return_value['neural'] + assert legacy.digest_text.call_args.args[0] == ['one sentence', 'another'] + legacy.start_neural_recording.assert_called_once_with('language_system', 'fMRI') + assert score.attrs['compatibility']['match'] == 'full' From de389b6a4a90754014801396583bf0c53cf65e24 Mon Sep 17 00:00:00 2001 From: Kartik Pradeepan Date: Sun, 1 Nov 2026 18:02:18 -0500 Subject: [PATCH 4/9] fix(compat): register legacy recordings and tasks once per configuration Legacy language subjects append every start_neural_recording call, so a facade that re-sent the setup per passage registered the recording once per digest_text. LanguageModelAdapter now applies a recording or task configuration once until it changes or the adapter is reset, which is the call sequence a legacy benchmark makes. Only declared input streams reach the legacy model, and responses without a clock carry t_ms=None. --- brainscore_language/compat/unified_adapter.py | 31 +++++++++++++------ tests/test_unified_adapter.py | 29 +++++++++++++++++ 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/brainscore_language/compat/unified_adapter.py b/brainscore_language/compat/unified_adapter.py index df9977a9..fe765321 100644 --- a/brainscore_language/compat/unified_adapter.py +++ b/brainscore_language/compat/unified_adapter.py @@ -11,11 +11,13 @@ from typing import Any, Dict, Optional, Set from brainscore_core.model_interface import UnifiedModel, TaskContext +from brainscore_core.session_view import DeclaredInputSession from brainscore_core.streaming import StreamEvent from brainscore_core.streaming_helpers import ( _drain_stream_events, _drive_behavior_session_via_process, _drive_neural_session_via_process, + measurement_settings, ) @@ -26,6 +28,11 @@ def __init__(self, legacy_model): self._task_context: Optional[TaskContext] = None self._recording_active: bool = False self._task_active: bool = False + # Setup the legacy model currently has. Legacy subjects append every + # recording registration, so a configuration is applied once until + # it changes or the adapter is reset. + self._registered_recordings: set = set() + self._configured_task = None @property def identifier(self) -> str: @@ -143,20 +150,21 @@ def interact(self, session) -> None: ``digest_text`` call that can return neural and behavioral outputs together. Other sessions use the typed drivers over ``process``. """ + session = DeclaredInputSession(session, self.in_channels) channels = tuple(session.requested_output_channels or ()) missing = set(channels) - self.out_channels if not channels or missing: raise ValueError(f'Undeclared output channels {sorted(missing)}; declared: ' f'{sorted(self.out_channels)}') context = session.task_context - settings = dict(getattr(context, 'metadata', None) or {}) + settings = measurement_settings(session) if 'legacy_text' not in settings: if channels == ('behavior',): _drive_behavior_session_via_process(self, session) else: _drive_neural_session_via_process(self, session) return - if context.task_type != 'passive': + if context is not None and context.task_type != 'passive': self.start_task(context) for channel in channels: if channel.startswith('neural:'): @@ -166,7 +174,7 @@ def interact(self, session) -> None: result = self._legacy.digest_text(settings['legacy_text']) for channel in channels: key = 'neural' if channel.startswith('neural:') else 'behavior' - session.respond(StreamEvent(channel, result[key], 0.0, meta={'driver': 'legacy'})) + session.respond(StreamEvent(channel, result[key], None, meta={'driver': 'legacy'})) # Backwards-compatible: existing benchmarks call digest_text() directly def digest_text(self, text): @@ -174,27 +182,30 @@ def digest_text(self, text): def start_task(self, task_context: TaskContext) -> None: self._task_context = task_context - self._task_active = True # Legacy ArtificialSubject.start_behavioral_task(task) takes ONE arg - self._legacy.start_behavioral_task(task_context.task_type) + self.start_behavioral_task(task_context.task_type) # Backwards-compatible: existing benchmarks call these directly def start_behavioral_task(self, task): self._task_active = True + if task == self._configured_task: + return self._legacy.start_behavioral_task(task) + self._configured_task = task def start_neural_recording(self, recording_target, recording_type='fMRI'): self._recording_active = True + registration = (recording_target, recording_type) + if registration in self._registered_recordings: + return # legacy subjects append; the benchmark registered this once self._legacy.start_neural_recording(recording_target, recording_type) + self._registered_recordings.add(registration) def start_recording(self, recording_target: str, time_bins=None, recording_type=None, **kwargs) -> None: - self._recording_active = True # Legacy ArtificialSubject.start_neural_recording(target, recording_type) # Default recording_type for language is 'fMRI' - self._legacy.start_neural_recording( - recording_target, recording_type or 'fMRI' - ) + self.start_neural_recording(recording_target, recording_type or 'fMRI') def reset(self) -> None: reset = getattr(self._legacy, 'reset', None) @@ -215,6 +226,8 @@ def reset(self) -> None: self._task_context = None self._recording_active = False self._task_active = False + self._registered_recordings = set() + self._configured_task = None if hasattr(self._legacy, 'current_tokens'): self._legacy.current_tokens = None diff --git a/tests/test_unified_adapter.py b/tests/test_unified_adapter.py index a17eb6ba..ab074704 100644 --- a/tests/test_unified_adapter.py +++ b/tests/test_unified_adapter.py @@ -463,3 +463,32 @@ def __call__(self, candidate): assert legacy.digest_text.call_args.args[0] == ['one sentence', 'another'] legacy.start_neural_recording.assert_called_once_with('language_system', 'fMRI') assert score.attrs['compatibility']['match'] == 'full' + + +def test_recording_registration_is_applied_once_per_configuration(): + """Legacy subjects append registrations; a benchmark registers once, so does the adapter.""" + from brainscore_core.legacy_facade import LegacyBenchmarkFacade + legacy = _make_legacy_model(region_layer_mapping={'language_system': 'language'}) + legacy.digest_text.return_value = {'neural': _language_neural_assembly()} + adapter = LanguageModelAdapter(legacy) + facade = LegacyBenchmarkFacade(adapter) + facade.start_neural_recording('language_system', recording_type='fMRI') + facade.digest_text(['passage one']) + facade.digest_text(['passage two']) + legacy.start_neural_recording.assert_called_once_with('language_system', 'fMRI') + assert legacy.digest_text.call_count == 2 + adapter.start_neural_recording('language_system', 'spikes') # a new registration + assert legacy.start_neural_recording.call_count == 2 + adapter.reset() + adapter.start_neural_recording('language_system', 'fMRI') + assert legacy.start_neural_recording.call_count == 3 + + +def test_behavioral_task_is_applied_once_per_configuration(): + legacy = _make_legacy_model(region_layer_mapping={'language_system': 'language'}) + adapter = LanguageModelAdapter(legacy) + adapter.start_behavioral_task('reading_times') + adapter.start_behavioral_task('reading_times') + legacy.start_behavioral_task.assert_called_once_with('reading_times') + adapter.start_behavioral_task('next_word') + assert legacy.start_behavioral_task.call_count == 2 From 62e2dcfe17daa81ecdf939f6e075d9edd33389ca Mon Sep 17 00:00:00 2001 From: Kartik Pradeepan Date: Sun, 1 Nov 2026 18:03:24 -0500 Subject: [PATCH 5/9] chore(ci): check the facade candidate against its pinned peers The coordinated workflow reads peer revisions from the pinned unified commit, so it now points at the unified revision whose install/peer-revisions.json names the facade branch tips; the requirements pin follows the same core revision. Re-pin to merged revisions before merging. --- .github/workflows/umi-v2.yml | 4 ++-- requirements-v2.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/umi-v2.yml b/.github/workflows/umi-v2.yml index 0d70508c..ae91822e 100644 --- a/.github/workflows/umi-v2.yml +++ b/.github/workflows/umi-v2.yml @@ -11,9 +11,9 @@ jobs: coordinated: name: UMI v2 # Update the workflow and runner revision together. - uses: KartikP/brainscore-unified/.github/workflows/offline-tests.yml@a95ad140a226cf5dd56ca9559e2504214078f2c1 + uses: KartikP/brainscore-unified/.github/workflows/offline-tests.yml@6b7485f3a8d52a9db5a2eedfc193900d0f19b0ed with: candidate: language candidate_repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} candidate_revision: ${{ github.event.pull_request.head.sha || github.sha }} - unified_revision: a95ad140a226cf5dd56ca9559e2504214078f2c1 + unified_revision: 6b7485f3a8d52a9db5a2eedfc193900d0f19b0ed diff --git a/requirements-v2.txt b/requirements-v2.txt index ee55b05d..8b7a212e 100644 --- a/requirements-v2.txt +++ b/requirements-v2.txt @@ -1,2 +1,2 @@ # Unpublished v2 core, pinned for standalone CI and documentation builds. -brainscore-core @ git+https://github.com/brain-score/core.git@434eb311fca090567653716ce1cb337c35646b1b +brainscore-core @ git+https://github.com/brain-score/core.git@dd92c84b5e543c8278d2efa833041f9b9b9c59c7 From 02aff759089ee17bad5f0d93da437cb90c5e8fb5 Mon Sep 17 00:00:00 2001 From: Kartik Pradeepan Date: Sun, 1 Nov 2026 19:00:18 -0500 Subject: [PATCH 6/9] chore(ci): re-pin peers after the refit and replay fixes --- .github/workflows/umi-v2.yml | 4 ++-- requirements-v2.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/umi-v2.yml b/.github/workflows/umi-v2.yml index ae91822e..9445a1c8 100644 --- a/.github/workflows/umi-v2.yml +++ b/.github/workflows/umi-v2.yml @@ -11,9 +11,9 @@ jobs: coordinated: name: UMI v2 # Update the workflow and runner revision together. - uses: KartikP/brainscore-unified/.github/workflows/offline-tests.yml@6b7485f3a8d52a9db5a2eedfc193900d0f19b0ed + uses: KartikP/brainscore-unified/.github/workflows/offline-tests.yml@b8880276242296c6857363777b4bd12a02ab5213 with: candidate: language candidate_repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} candidate_revision: ${{ github.event.pull_request.head.sha || github.sha }} - unified_revision: 6b7485f3a8d52a9db5a2eedfc193900d0f19b0ed + unified_revision: b8880276242296c6857363777b4bd12a02ab5213 diff --git a/requirements-v2.txt b/requirements-v2.txt index 8b7a212e..376870cb 100644 --- a/requirements-v2.txt +++ b/requirements-v2.txt @@ -1,2 +1,2 @@ # Unpublished v2 core, pinned for standalone CI and documentation builds. -brainscore-core @ git+https://github.com/brain-score/core.git@dd92c84b5e543c8278d2efa833041f9b9b9c59c7 +brainscore-core @ git+https://github.com/brain-score/core.git@15dcc3c6cbfea16565052abd2663b8de4bf1b203 From ac7d2b100a6a25f8108c35d2482a54e72788454c Mon Sep 17 00:00:00 2001 From: Kartik Pradeepan Date: Mon, 2 Nov 2026 09:40:31 -0500 Subject: [PATCH 7/9] chore(ci): pin merged core and coordinated v3 checks --- .github/workflows/umi-v2.yml | 4 ++-- requirements-v2.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/umi-v2.yml b/.github/workflows/umi-v2.yml index 9445a1c8..21f91dd6 100644 --- a/.github/workflows/umi-v2.yml +++ b/.github/workflows/umi-v2.yml @@ -11,9 +11,9 @@ jobs: coordinated: name: UMI v2 # Update the workflow and runner revision together. - uses: KartikP/brainscore-unified/.github/workflows/offline-tests.yml@b8880276242296c6857363777b4bd12a02ab5213 + uses: KartikP/brainscore-unified/.github/workflows/offline-tests.yml@42994ca24c05b6b2547e0f7153b2018da54d915e with: candidate: language candidate_repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} candidate_revision: ${{ github.event.pull_request.head.sha || github.sha }} - unified_revision: b8880276242296c6857363777b4bd12a02ab5213 + unified_revision: 42994ca24c05b6b2547e0f7153b2018da54d915e diff --git a/requirements-v2.txt b/requirements-v2.txt index 376870cb..929e61fe 100644 --- a/requirements-v2.txt +++ b/requirements-v2.txt @@ -1,2 +1,2 @@ # Unpublished v2 core, pinned for standalone CI and documentation builds. -brainscore-core @ git+https://github.com/brain-score/core.git@15dcc3c6cbfea16565052abd2663b8de4bf1b203 +brainscore-core @ git+https://github.com/brain-score/core.git@0fbb419ff0ba461e6b1e217c845e0fc61d8096a1 From 7d3ca103eec874b2eb600bb399d3d33364f0ba49 Mon Sep 17 00:00:00 2001 From: Kartik Pradeepan Date: Mon, 2 Nov 2026 10:02:33 -0500 Subject: [PATCH 8/9] fix(adapter): declare legacy embedding recording outputs --- brainscore_language/compat/unified_adapter.py | 31 ++++++- tests/test_unified_adapter.py | 84 +++++++++++++++++++ 2 files changed, 113 insertions(+), 2 deletions(-) diff --git a/brainscore_language/compat/unified_adapter.py b/brainscore_language/compat/unified_adapter.py index fe765321..15dbdd0f 100644 --- a/brainscore_language/compat/unified_adapter.py +++ b/brainscore_language/compat/unified_adapter.py @@ -21,10 +21,36 @@ ) +# These legacy registrations expose the same embedding at each recording target, +# without a region-to-layer map. Declare their outputs here, leaving the plugins +# unchanged. They cannot answer behavioral tasks. +_EMBEDDING_OUTPUT_CHANNELS = frozenset({ + 'neural:language_system', + 'neural:language_system_left_hemisphere', + 'neural:language_system_right_hemisphere', +}) +_LEGACY_OUTPUT_CHANNELS = { + 'randomembedding-100': _EMBEDDING_OUTPUT_CHANNELS, + 'randomembedding-1600': _EMBEDDING_OUTPUT_CHANNELS, + 'glove-840b': _EMBEDDING_OUTPUT_CHANNELS, +} + + class LanguageModelAdapter(UnifiedModel): + """Adapt a legacy subject, with explicit outputs when no layer map exists. + + Known embedding registrations have declarations above. Custom legacy subjects + can supply ``out_channels``; otherwise the public layer map and behavioral + compatibility path supply the existing defaults. + """ - def __init__(self, legacy_model): + def __init__(self, legacy_model: Any, *, + out_channels: Optional[Set[str]] = None) -> None: self._legacy = legacy_model + self._declared_out_channels = ( + frozenset(out_channels) if out_channels is not None + else _LEGACY_OUTPUT_CHANNELS.get(self.identifier) + ) self._task_context: Optional[TaskContext] = None self._recording_active: bool = False self._task_active: bool = False @@ -139,7 +165,8 @@ def _process_table(self, stimuli): @property def out_channels(self) -> Set[str]: - # A legacy ArtificialSubject answers behavioral tasks as well as recording. + if self._declared_out_channels is not None: + return set(self._declared_out_channels) return super().out_channels | {'behavior'} def interact(self, session) -> None: diff --git a/tests/test_unified_adapter.py b/tests/test_unified_adapter.py index ab074704..da1ed52d 100644 --- a/tests/test_unified_adapter.py +++ b/tests/test_unified_adapter.py @@ -115,6 +115,90 @@ def test_region_layer_map_empty_when_no_mapping(self): adapter = LanguageModelAdapter(legacy) assert adapter.region_layer_map == {} + @pytest.mark.parametrize('identifier', [ + 'randomembedding-100', 'randomembedding-1600', 'glove-840b', + ]) + def test_embedding_declarations_without_a_layer_map(self, identifier): + adapter = LanguageModelAdapter(_make_legacy_model(identifier=identifier)) + assert adapter.out_channels == { + 'neural:language_system', + 'neural:language_system_left_hemisphere', + 'neural:language_system_right_hemisphere', + } + assert adapter.region_layer_map == {} + + def test_explicit_output_declaration_is_copied(self): + channels = {'neural:custom_region'} + adapter = LanguageModelAdapter(_make_legacy_model(), out_channels=channels) + channels.add('behavior') + assert adapter.out_channels == {'neural:custom_region'} + returned = adapter.out_channels + returned.add('behavior') + assert adapter.out_channels == {'neural:custom_region'} + + def test_empty_output_declaration_is_respected(self): + adapter = LanguageModelAdapter(_make_legacy_model(), out_channels=set()) + assert adapter.out_channels == set() + + def test_unmapped_behavior_model_does_not_gain_neural_outputs(self): + adapter = LanguageModelAdapter(_make_legacy_model(identifier='earley-parser')) + assert adapter.out_channels == {'behavior'} + + +class TestEmbeddingSessions: + + @pytest.mark.parametrize('identifier', [ + 'randomembedding-100', 'randomembedding-1600', + ]) + @pytest.mark.parametrize('target', [ + 'language_system', + 'language_system_left_hemisphere', + 'language_system_right_hemisphere', + ]) + def test_loaded_embedding_facade_matches_legacy_exactly( + self, identifier, target, monkeypatch): + from brainscore_language import load_model + from brainscore_core.legacy_facade import LegacyBenchmarkFacade + + monkeypatch.setenv('BS_INSTALL_DEPENDENCIES', 'no') + adapter = load_model(identifier) + legacy = load_model(identifier)._legacy + text = ['the quick brown', 'fox jumps'] + legacy.start_neural_recording(target, 'fMRI') + expected = legacy.digest_text(text)['neural'] + + facade = LegacyBenchmarkFacade(adapter) + facade.start_neural_recording(target, 'fMRI') + for _ in range(2): + actual = facade.digest_text(text)['neural'] + xr.testing.assert_identical(actual, expected) + assert adapter._legacy.neural_recordings == [(target, 'fMRI')] + + def test_loaded_embedding_typed_session_matches_legacy(self, monkeypatch): + from brainscore_language import load_model + + monkeypatch.setenv('BS_INSTALL_DEPENDENCIES', 'no') + adapter = load_model('randomembedding-100') + stimuli = _language_stimulus_set() + legacy = load_model('randomembedding-100')._legacy + legacy.start_neural_recording('language_system', 'fMRI') + expected = legacy.digest_text(list(stimuli['sentence']))['neural'] + + actual = neural_response(adapter, stimuli, record='language_system') + np.testing.assert_array_equal(actual.values, expected.values) + assert list(actual['stimulus_id'].values) == list(stimuli['stimulus_id']) + assert list(actual['object_name'].values) == list(stimuli['object_name']) + + def test_embedding_rejects_behavior_before_compute(self): + from brainscore_core.legacy_facade import LegacyBenchmarkFacade + + legacy = _make_legacy_model(identifier='randomembedding-100') + facade = LegacyBenchmarkFacade(LanguageModelAdapter(legacy)) + with pytest.raises(NotImplementedError, match="no 'behavior' output"): + facade.start_behavioral_task('next_word') + legacy.start_behavioral_task.assert_not_called() + legacy.digest_text.assert_not_called() + class TestLanguageAdapterProcess: From 5c458749e995f1e9e3f8e04e1da22d34aeaf7313 Mon Sep 17 00:00:00 2001 From: Kartik Pradeepan Date: Mon, 2 Nov 2026 10:14:06 -0500 Subject: [PATCH 9/9] test(submission): assert facade rejection diagnostics --- tests/test_submission/test_endpoints.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_submission/test_endpoints.py b/tests/test_submission/test_endpoints.py index 8985fe6d..ab37a04e 100644 --- a/tests/test_submission/test_endpoints.py +++ b/tests/test_submission/test_endpoints.py @@ -85,4 +85,6 @@ def test_model_cannot_run(self): assert score_entry.score_raw is None assert score_entry.error is None assert 'NotImplementedError' in score_entry.comment - assert 'embedding.py' in score_entry.comment # test for some stacktrace + assert "declares no 'behavior' output" in score_entry.comment + assert 'Traceback (most recent call last)' in score_entry.comment + assert 'legacy_facade.py' in score_entry.comment