diff --git a/.github/workflows/umi-v2.yml b/.github/workflows/umi-v2.yml index 0d70508c..21f91dd6 100644 --- a/.github/workflows/umi-v2.yml +++ b/.github/workflows/umi-v2.yml @@ -11,9 +11,9 @@ jobs: coordinated: name: UMI v2 # Update the workflow and runner revision together. - uses: KartikP/brainscore-unified/.github/workflows/offline-tests.yml@a95ad140a226cf5dd56ca9559e2504214078f2c1 + uses: KartikP/brainscore-unified/.github/workflows/offline-tests.yml@42994ca24c05b6b2547e0f7153b2018da54d915e with: candidate: language candidate_repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} candidate_revision: ${{ github.event.pull_request.head.sha || github.sha }} - unified_revision: a95ad140a226cf5dd56ca9559e2504214078f2c1 + unified_revision: 42994ca24c05b6b2547e0f7153b2018da54d915e diff --git a/brainscore_language/__init__.py b/brainscore_language/__init__.py index 9052ab6b..22e1357d 100644 --- a/brainscore_language/__init__.py +++ b/brainscore_language/__init__.py @@ -37,8 +37,9 @@ def load_benchmark(identifier: str) -> Benchmark: import_plugin('brainscore_language', 'benchmarks', identifier) from brainscore_core.compatibility import ensure_legacy_benchmark_modalities + from brainscore_core.legacy_facade import mark_legacy_benchmark benchmark = benchmark_registry[identifier]() - return ensure_legacy_benchmark_modalities(benchmark, {'text'}) + return mark_legacy_benchmark(ensure_legacy_benchmark_modalities(benchmark, {'text'})) def load_model(identifier: str) -> 'Subject': @@ -60,7 +61,6 @@ def _run_score(model_identifier: str, benchmark_identifier: str, Score the model referenced by the `model_identifier` on the benchmark referenced by the `benchmark_identifier`. """ from brainscore_core.compatibility import ( - check_channel_compatibility, check_compatibility, ) from brainscore_core.extraction_cache import weight_fingerprint_scope @@ -78,15 +78,19 @@ def _run_score(model_identifier: str, benchmark_identifier: str, requested_channels = requested_output_channels_for_score(benchmark) # Pre-flight checks - check_compatibility(model, benchmark) - check_channel_compatibility(model, benchmark) + compatibility = check_compatibility(model, benchmark) if check_mem: check_memory(model, benchmark) + # A legacy benchmark calls digest_text and friends; it receives the permanent + # facade, which runs those calls as sessions through the subject's interact. + from brainscore_core.legacy_facade import subject_for_benchmark + candidate = subject_for_benchmark(model, benchmark) + import time as _time _t0 = _time.time() with weight_fingerprint_scope(): - score: Score = benchmark(model) + score: Score = benchmark(candidate) score.attrs['runtime_sec'] = round(_time.time() - _t0, 2) score.attrs['model_identifier'] = model_identifier score.attrs['benchmark_identifier'] = benchmark_identifier @@ -97,6 +101,7 @@ def _run_score(model_identifier: str, benchmark_identifier: str, protocol=infer_score_protocol(model, benchmark, requested_channels), harness_id='brainscore_language', ) + score.attrs['compatibility'] = compatibility.as_attrs() return score diff --git a/brainscore_language/benchmarks/pereira2018/unified.py b/brainscore_language/benchmarks/pereira2018/unified.py index 10e25189..3de1983d 100644 --- a/brainscore_language/benchmarks/pereira2018/unified.py +++ b/brainscore_language/benchmarks/pereira2018/unified.py @@ -31,6 +31,10 @@ class _Pereira2018ExperimentUnified(_Pereira2018Experiment): NeuroidAssembly directly. """ + # Calls the typed convention on the candidate itself; scoring must not + # wrap the candidate in the legacy-benchmark facade. + subject_api = 'typed' + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._identifier = self.identifier + '-unified' diff --git a/brainscore_language/compat/unified_adapter.py b/brainscore_language/compat/unified_adapter.py index 2245dc34..15dbdd0f 100644 --- a/brainscore_language/compat/unified_adapter.py +++ b/brainscore_language/compat/unified_adapter.py @@ -11,16 +11,54 @@ from typing import Any, Dict, Optional, Set from brainscore_core.model_interface import UnifiedModel, TaskContext -from brainscore_core.streaming_helpers import _drive_neural_session_via_process +from brainscore_core.session_view import DeclaredInputSession +from brainscore_core.streaming import StreamEvent +from brainscore_core.streaming_helpers import ( + _drain_stream_events, + _drive_behavior_session_via_process, + _drive_neural_session_via_process, + measurement_settings, +) + + +# These legacy registrations expose the same embedding at each recording target, +# without a region-to-layer map. Declare their outputs here, leaving the plugins +# unchanged. They cannot answer behavioral tasks. +_EMBEDDING_OUTPUT_CHANNELS = frozenset({ + 'neural:language_system', + 'neural:language_system_left_hemisphere', + 'neural:language_system_right_hemisphere', +}) +_LEGACY_OUTPUT_CHANNELS = { + 'randomembedding-100': _EMBEDDING_OUTPUT_CHANNELS, + 'randomembedding-1600': _EMBEDDING_OUTPUT_CHANNELS, + 'glove-840b': _EMBEDDING_OUTPUT_CHANNELS, +} class LanguageModelAdapter(UnifiedModel): + """Adapt a legacy subject, with explicit outputs when no layer map exists. - def __init__(self, legacy_model): + Known embedding registrations have declarations above. Custom legacy subjects + can supply ``out_channels``; otherwise the public layer map and behavioral + compatibility path supply the existing defaults. + """ + + def __init__(self, legacy_model: Any, *, + out_channels: Optional[Set[str]] = None) -> None: self._legacy = legacy_model + self._declared_out_channels = ( + frozenset(out_channels) if out_channels is not None + else _LEGACY_OUTPUT_CHANNELS.get(self.identifier) + ) self._task_context: Optional[TaskContext] = None self._recording_active: bool = False self._task_active: bool = False + # Setup the legacy model currently has. Legacy subjects append every + # recording registration, so a configuration is applied once until + # it changes or the adapter is reset. + self._registered_recordings: set = set() + self._configured_task = None @property def identifier(self) -> str: @@ -125,8 +163,45 @@ def _process_table(self, stimuli): return xr.concat(outputs, dim='presentation').isel( presentation=np.argsort(positions_out)) + @property + def out_channels(self) -> Set[str]: + if self._declared_out_channels is not None: + return set(self._declared_out_channels) + return super().out_channels | {'behavior'} + def interact(self, session) -> None: - _drive_neural_session_via_process(self, session) + """Run a session; a legacy-benchmark session replays the exact ``digest_text``. + + A session carrying ``legacy_text`` (from the legacy-benchmark facade) + configures the legacy model as the benchmark did and makes one + ``digest_text`` call that can return neural and behavioral outputs + together. Other sessions use the typed drivers over ``process``. + """ + session = DeclaredInputSession(session, self.in_channels) + channels = tuple(session.requested_output_channels or ()) + missing = set(channels) - self.out_channels + if not channels or missing: + raise ValueError(f'Undeclared output channels {sorted(missing)}; declared: ' + f'{sorted(self.out_channels)}') + context = session.task_context + settings = measurement_settings(session) + if 'legacy_text' not in settings: + if channels == ('behavior',): + _drive_behavior_session_via_process(self, session) + else: + _drive_neural_session_via_process(self, session) + return + if context is not None and context.task_type != 'passive': + self.start_task(context) + for channel in channels: + if channel.startswith('neural:'): + self.start_neural_recording(channel.split(':', 1)[1], + settings.get('recording_type', 'fMRI')) + _drain_stream_events(session) + result = self._legacy.digest_text(settings['legacy_text']) + for channel in channels: + key = 'neural' if channel.startswith('neural:') else 'behavior' + session.respond(StreamEvent(channel, result[key], None, meta={'driver': 'legacy'})) # Backwards-compatible: existing benchmarks call digest_text() directly def digest_text(self, text): @@ -134,27 +209,30 @@ def digest_text(self, text): def start_task(self, task_context: TaskContext) -> None: self._task_context = task_context - self._task_active = True # Legacy ArtificialSubject.start_behavioral_task(task) takes ONE arg - self._legacy.start_behavioral_task(task_context.task_type) + self.start_behavioral_task(task_context.task_type) # Backwards-compatible: existing benchmarks call these directly def start_behavioral_task(self, task): self._task_active = True + if task == self._configured_task: + return self._legacy.start_behavioral_task(task) + self._configured_task = task def start_neural_recording(self, recording_target, recording_type='fMRI'): self._recording_active = True + registration = (recording_target, recording_type) + if registration in self._registered_recordings: + return # legacy subjects append; the benchmark registered this once self._legacy.start_neural_recording(recording_target, recording_type) + self._registered_recordings.add(registration) def start_recording(self, recording_target: str, time_bins=None, recording_type=None, **kwargs) -> None: - self._recording_active = True # Legacy ArtificialSubject.start_neural_recording(target, recording_type) # Default recording_type for language is 'fMRI' - self._legacy.start_neural_recording( - recording_target, recording_type or 'fMRI' - ) + self.start_neural_recording(recording_target, recording_type or 'fMRI') def reset(self) -> None: reset = getattr(self._legacy, 'reset', None) @@ -175,9 +253,16 @@ def reset(self) -> None: self._task_context = None self._recording_active = False self._task_active = False + self._registered_recordings = set() + self._configured_task = None if hasattr(self._legacy, 'current_tokens'): self._legacy.current_tokens = None - def __getattr__(self, name): - # Delegate attribute access to the legacy model for backwards compatibility - return getattr(self._legacy, name) + @property + def legacy_model(self): + """The wrapped legacy model, for tools that need its own attributes. + + Unknown attributes are no longer forwarded implicitly: a benchmark + sees only what the adapter declares (Contract Rules v3, rule 2.1). + """ + return self._legacy diff --git a/requirements-v2.txt b/requirements-v2.txt index ee55b05d..929e61fe 100644 --- a/requirements-v2.txt +++ b/requirements-v2.txt @@ -1,2 +1,2 @@ # Unpublished v2 core, pinned for standalone CI and documentation builds. -brainscore-core @ git+https://github.com/brain-score/core.git@434eb311fca090567653716ce1cb337c35646b1b +brainscore-core @ git+https://github.com/brain-score/core.git@0fbb419ff0ba461e6b1e217c845e0fc61d8096a1 diff --git a/tests/test_legacy_score.py b/tests/test_legacy_score.py index ea952d5b..ddaffe3e 100644 --- a/tests/test_legacy_score.py +++ b/tests/test_legacy_score.py @@ -13,7 +13,8 @@ class LegacyBenchmark: parent = 'behavior' def __call__(self, model): - assert model.required_modalities == {'text'} + # The benchmark receives the legacy facade; the subject is behind it. + assert model.subject.required_channels == {"text"} return Score(0.5) diff --git a/tests/test_preflight.py b/tests/test_preflight.py index 3d013353..8058cdd7 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -45,7 +45,7 @@ def test_incompatible_model_raises_compatibility_error( mock_load_model.return_value = model mock_load_benchmark.return_value = benchmark - with pytest.raises(CompatibilityError, match="does not support modalities required"): + with pytest.raises(CompatibilityError, match="takes in none of the inputs"): _run_score('test-model', 'test-bench', check_mem=False) @patch('brainscore_language.load_benchmark') @@ -95,6 +95,7 @@ def test_score_metadata_attrs_are_stamped( assert score.attrs['out_channels'] == ('behavior',) assert score.attrs['protocol'] == 'behavior:readout' assert score.attrs['subject_id'] == 'test-model' + assert score.attrs['compatibility']['match'] == 'full' assert score.attrs['harness_id'] == 'brainscore_language' @patch('brainscore_core.compatibility.check_channel_compatibility') diff --git a/tests/test_submission/test_endpoints.py b/tests/test_submission/test_endpoints.py index 8985fe6d..ab37a04e 100644 --- a/tests/test_submission/test_endpoints.py +++ b/tests/test_submission/test_endpoints.py @@ -85,4 +85,6 @@ def test_model_cannot_run(self): assert score_entry.score_raw is None assert score_entry.error is None assert 'NotImplementedError' in score_entry.comment - assert 'embedding.py' in score_entry.comment # test for some stacktrace + assert "declares no 'behavior' output" in score_entry.comment + assert 'Traceback (most recent call last)' in score_entry.comment + assert 'legacy_facade.py' in score_entry.comment diff --git a/tests/test_unified_adapter.py b/tests/test_unified_adapter.py index e216f0fc..da1ed52d 100644 --- a/tests/test_unified_adapter.py +++ b/tests/test_unified_adapter.py @@ -115,6 +115,90 @@ def test_region_layer_map_empty_when_no_mapping(self): adapter = LanguageModelAdapter(legacy) assert adapter.region_layer_map == {} + @pytest.mark.parametrize('identifier', [ + 'randomembedding-100', 'randomembedding-1600', 'glove-840b', + ]) + def test_embedding_declarations_without_a_layer_map(self, identifier): + adapter = LanguageModelAdapter(_make_legacy_model(identifier=identifier)) + assert adapter.out_channels == { + 'neural:language_system', + 'neural:language_system_left_hemisphere', + 'neural:language_system_right_hemisphere', + } + assert adapter.region_layer_map == {} + + def test_explicit_output_declaration_is_copied(self): + channels = {'neural:custom_region'} + adapter = LanguageModelAdapter(_make_legacy_model(), out_channels=channels) + channels.add('behavior') + assert adapter.out_channels == {'neural:custom_region'} + returned = adapter.out_channels + returned.add('behavior') + assert adapter.out_channels == {'neural:custom_region'} + + def test_empty_output_declaration_is_respected(self): + adapter = LanguageModelAdapter(_make_legacy_model(), out_channels=set()) + assert adapter.out_channels == set() + + def test_unmapped_behavior_model_does_not_gain_neural_outputs(self): + adapter = LanguageModelAdapter(_make_legacy_model(identifier='earley-parser')) + assert adapter.out_channels == {'behavior'} + + +class TestEmbeddingSessions: + + @pytest.mark.parametrize('identifier', [ + 'randomembedding-100', 'randomembedding-1600', + ]) + @pytest.mark.parametrize('target', [ + 'language_system', + 'language_system_left_hemisphere', + 'language_system_right_hemisphere', + ]) + def test_loaded_embedding_facade_matches_legacy_exactly( + self, identifier, target, monkeypatch): + from brainscore_language import load_model + from brainscore_core.legacy_facade import LegacyBenchmarkFacade + + monkeypatch.setenv('BS_INSTALL_DEPENDENCIES', 'no') + adapter = load_model(identifier) + legacy = load_model(identifier)._legacy + text = ['the quick brown', 'fox jumps'] + legacy.start_neural_recording(target, 'fMRI') + expected = legacy.digest_text(text)['neural'] + + facade = LegacyBenchmarkFacade(adapter) + facade.start_neural_recording(target, 'fMRI') + for _ in range(2): + actual = facade.digest_text(text)['neural'] + xr.testing.assert_identical(actual, expected) + assert adapter._legacy.neural_recordings == [(target, 'fMRI')] + + def test_loaded_embedding_typed_session_matches_legacy(self, monkeypatch): + from brainscore_language import load_model + + monkeypatch.setenv('BS_INSTALL_DEPENDENCIES', 'no') + adapter = load_model('randomembedding-100') + stimuli = _language_stimulus_set() + legacy = load_model('randomembedding-100')._legacy + legacy.start_neural_recording('language_system', 'fMRI') + expected = legacy.digest_text(list(stimuli['sentence']))['neural'] + + actual = neural_response(adapter, stimuli, record='language_system') + np.testing.assert_array_equal(actual.values, expected.values) + assert list(actual['stimulus_id'].values) == list(stimuli['stimulus_id']) + assert list(actual['object_name'].values) == list(stimuli['object_name']) + + def test_embedding_rejects_behavior_before_compute(self): + from brainscore_core.legacy_facade import LegacyBenchmarkFacade + + legacy = _make_legacy_model(identifier='randomembedding-100') + facade = LegacyBenchmarkFacade(LanguageModelAdapter(legacy)) + with pytest.raises(NotImplementedError, match="no 'behavior' output"): + facade.start_behavioral_task('next_word') + legacy.start_behavioral_task.assert_not_called() + legacy.digest_text.assert_not_called() + class TestLanguageAdapterProcess: @@ -405,3 +489,90 @@ def test_load_model_does_not_double_wrap_unified(self): assert model is native assert not isinstance(model, LanguageModelAdapter) + + +# --- legacy benchmarks through the permanent facade ----------------------- + +@pytest.mark.parametrize('neural,behavior', [(True, False), (False, True), (True, True)]) +def test_legacy_facade_preserves_language_outputs_in_one_call(neural, behavior): + from brainscore_core.legacy_facade import LegacyBenchmarkFacade + from brainscore_core.supported_data_standards.brainio.assemblies import BehavioralAssembly + legacy = _make_legacy_model(region_layer_mapping={'language_system': 'language'}) + expected = {} + if neural: + expected['neural'] = _language_neural_assembly() + if behavior: + expected['behavior'] = BehavioralAssembly(np.array([1., 2.]), dims=['presentation']) + legacy.digest_text.return_value = expected + facade = LegacyBenchmarkFacade(LanguageModelAdapter(legacy)) + if neural: + facade.start_neural_recording('language_system', recording_type='spikes') + if behavior: + facade.start_behavioral_task('reading_times') + text = np.array(['the quick brown', 'fox jumps']) + actual = facade.digest_text(text) + assert actual.keys() == expected.keys() + for key in expected: + assert actual[key] is expected[key] + assert legacy.digest_text.call_count == 1 + assert legacy.digest_text.call_args.args[0] is text + if neural: + legacy.start_neural_recording.assert_called_once_with('language_system', 'spikes') + if behavior: + legacy.start_behavioral_task.assert_called_once_with('reading_times') + + +def test_scoring_wraps_the_adapter_for_a_legacy_language_benchmark(monkeypatch): + import brainscore_language + from brainscore_core.metrics import Score + + class LegacyBenchmark: + identifier = 'legacy-language' + + def __call__(self, candidate): + candidate.start_neural_recording('language_system', 'fMRI') + output = candidate.digest_text(['one sentence', 'another']) + score = Score([1.0], coords={'aggregation': ['center']}, dims=['aggregation']) + score.attrs.update(candidate=type(candidate).__name__, output=output) + return score + + legacy = _make_legacy_model(region_layer_mapping={'language_system': 'language'}) + legacy.digest_text.return_value = {'neural': _language_neural_assembly()} + monkeypatch.setattr(brainscore_language, 'import_plugin', lambda *a, **k: None) + monkeypatch.setitem(brainscore_language.benchmark_registry, 'legacy-language', LegacyBenchmark) + monkeypatch.setattr(brainscore_language, 'load_model', lambda _: LanguageModelAdapter(legacy)) + score = brainscore_language._run_score('m', 'legacy-language', check_mem=False) + assert score.attrs['candidate'] == 'LegacyBenchmarkFacade' + assert score.attrs['output']['neural'] is legacy.digest_text.return_value['neural'] + assert legacy.digest_text.call_args.args[0] == ['one sentence', 'another'] + legacy.start_neural_recording.assert_called_once_with('language_system', 'fMRI') + assert score.attrs['compatibility']['match'] == 'full' + + +def test_recording_registration_is_applied_once_per_configuration(): + """Legacy subjects append registrations; a benchmark registers once, so does the adapter.""" + from brainscore_core.legacy_facade import LegacyBenchmarkFacade + legacy = _make_legacy_model(region_layer_mapping={'language_system': 'language'}) + legacy.digest_text.return_value = {'neural': _language_neural_assembly()} + adapter = LanguageModelAdapter(legacy) + facade = LegacyBenchmarkFacade(adapter) + facade.start_neural_recording('language_system', recording_type='fMRI') + facade.digest_text(['passage one']) + facade.digest_text(['passage two']) + legacy.start_neural_recording.assert_called_once_with('language_system', 'fMRI') + assert legacy.digest_text.call_count == 2 + adapter.start_neural_recording('language_system', 'spikes') # a new registration + assert legacy.start_neural_recording.call_count == 2 + adapter.reset() + adapter.start_neural_recording('language_system', 'fMRI') + assert legacy.start_neural_recording.call_count == 3 + + +def test_behavioral_task_is_applied_once_per_configuration(): + legacy = _make_legacy_model(region_layer_mapping={'language_system': 'language'}) + adapter = LanguageModelAdapter(legacy) + adapter.start_behavioral_task('reading_times') + adapter.start_behavioral_task('reading_times') + legacy.start_behavioral_task.assert_called_once_with('reading_times') + adapter.start_behavioral_task('next_word') + assert legacy.start_behavioral_task.call_count == 2