feat(adapter): preserve legacy language scoring through sessions

.github/workflows/umi-v2.yml

@@ -11,9 +11,9 @@ jobs:
   coordinated:
     name: UMI v2
     # Update the workflow and runner revision together.
-    uses: KartikP/brainscore-unified/.github/workflows/offline-tests.yml@a95ad140a226cf5dd56ca9559e2504214078f2c1
+    uses: KartikP/brainscore-unified/.github/workflows/offline-tests.yml@42994ca24c05b6b2547e0f7153b2018da54d915e
     with:
       candidate: language
       candidate_repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
       candidate_revision: ${{ github.event.pull_request.head.sha || github.sha }}
-      unified_revision: a95ad140a226cf5dd56ca9559e2504214078f2c1
+      unified_revision: 42994ca24c05b6b2547e0f7153b2018da54d915e

brainscore_language/__init__.py

@@ -37,8 +37,9 @@ def load_benchmark(identifier: str) -> Benchmark:
     import_plugin('brainscore_language', 'benchmarks', identifier)
 
     from brainscore_core.compatibility import ensure_legacy_benchmark_modalities
+    from brainscore_core.legacy_facade import mark_legacy_benchmark
     benchmark = benchmark_registry[identifier]()
-    return ensure_legacy_benchmark_modalities(benchmark, {'text'})
+    return mark_legacy_benchmark(ensure_legacy_benchmark_modalities(benchmark, {'text'}))
 
 
 def load_model(identifier: str) -> 'Subject':
@@ -60,7 +61,6 @@ def _run_score(model_identifier: str, benchmark_identifier: str,
     Score the model referenced by the `model_identifier` on the benchmark referenced by the `benchmark_identifier`.
     """
     from brainscore_core.compatibility import (
-        check_channel_compatibility,
         check_compatibility,
     )
     from brainscore_core.extraction_cache import weight_fingerprint_scope
@@ -78,15 +78,19 @@ def _run_score(model_identifier: str, benchmark_identifier: str,
     requested_channels = requested_output_channels_for_score(benchmark)
 
     # Pre-flight checks
-    check_compatibility(model, benchmark)
-    check_channel_compatibility(model, benchmark)
+    compatibility = check_compatibility(model, benchmark)
     if check_mem:
         check_memory(model, benchmark)
 
+    # A legacy benchmark calls digest_text and friends; it receives the permanent
+    # facade, which runs those calls as sessions through the subject's interact.
+    from brainscore_core.legacy_facade import subject_for_benchmark
+    candidate = subject_for_benchmark(model, benchmark)
+
     import time as _time
     _t0 = _time.time()
     with weight_fingerprint_scope():
-        score: Score = benchmark(model)
+        score: Score = benchmark(candidate)
     score.attrs['runtime_sec'] = round(_time.time() - _t0, 2)
     score.attrs['model_identifier'] = model_identifier
     score.attrs['benchmark_identifier'] = benchmark_identifier
@@ -97,6 +101,7 @@ def _run_score(model_identifier: str, benchmark_identifier: str,
         protocol=infer_score_protocol(model, benchmark, requested_channels),
         harness_id='brainscore_language',
     )
+    score.attrs['compatibility'] = compatibility.as_attrs()
     return score
 
 

brainscore_language/benchmarks/pereira2018/unified.py

@@ -31,6 +31,10 @@ class _Pereira2018ExperimentUnified(_Pereira2018Experiment):
       NeuroidAssembly directly.
     """
 
+    # Calls the typed convention on the candidate itself; scoring must not
+    # wrap the candidate in the legacy-benchmark facade.
+    subject_api = 'typed'
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._identifier = self.identifier + '-unified'

brainscore_language/compat/unified_adapter.py

@@ -11,16 +11,54 @@
 from typing import Any, Dict, Optional, Set
 
 from brainscore_core.model_interface import UnifiedModel, TaskContext
-from brainscore_core.streaming_helpers import _drive_neural_session_via_process
+from brainscore_core.session_view import DeclaredInputSession
+from brainscore_core.streaming import StreamEvent
+from brainscore_core.streaming_helpers import (
+    _drain_stream_events,
+    _drive_behavior_session_via_process,
+    _drive_neural_session_via_process,
+    measurement_settings,
+)
+
+
+# These legacy registrations expose the same embedding at each recording target,
+# without a region-to-layer map. Declare their outputs here, leaving the plugins
+# unchanged. They cannot answer behavioral tasks.
+_EMBEDDING_OUTPUT_CHANNELS = frozenset({
+    'neural:language_system',
+    'neural:language_system_left_hemisphere',
+    'neural:language_system_right_hemisphere',
+})
+_LEGACY_OUTPUT_CHANNELS = {
+    'randomembedding-100': _EMBEDDING_OUTPUT_CHANNELS,
+    'randomembedding-1600': _EMBEDDING_OUTPUT_CHANNELS,
+    'glove-840b': _EMBEDDING_OUTPUT_CHANNELS,
+}
 
 
 class LanguageModelAdapter(UnifiedModel):
+    """Adapt a legacy subject, with explicit outputs when no layer map exists.
 
-    def __init__(self, legacy_model):
+    Known embedding registrations have declarations above. Custom legacy subjects
+    can supply ``out_channels``; otherwise the public layer map and behavioral
+    compatibility path supply the existing defaults.
+    """
+
+    def __init__(self, legacy_model: Any, *,
+                 out_channels: Optional[Set[str]] = None) -> None:
         self._legacy = legacy_model
+        self._declared_out_channels = (
+            frozenset(out_channels) if out_channels is not None
+            else _LEGACY_OUTPUT_CHANNELS.get(self.identifier)
+        )
         self._task_context: Optional[TaskContext] = None
         self._recording_active: bool = False
         self._task_active: bool = False
+        # Setup the legacy model currently has. Legacy subjects append every
+        # recording registration, so a configuration is applied once until
+        # it changes or the adapter is reset.
+        self._registered_recordings: set = set()
+        self._configured_task = None
 
     @property
     def identifier(self) -> str:
@@ -125,36 +163,76 @@ def _process_table(self, stimuli):
         return xr.concat(outputs, dim='presentation').isel(
             presentation=np.argsort(positions_out))
 
+    @property
+    def out_channels(self) -> Set[str]:
+        if self._declared_out_channels is not None:
+            return set(self._declared_out_channels)
+        return super().out_channels | {'behavior'}
+
     def interact(self, session) -> None:
-        _drive_neural_session_via_process(self, session)
+        """Run a session; a legacy-benchmark session replays the exact ``digest_text``.
+
+        A session carrying ``legacy_text`` (from the legacy-benchmark facade)
+        configures the legacy model as the benchmark did and makes one
+        ``digest_text`` call that can return neural and behavioral outputs
+        together. Other sessions use the typed drivers over ``process``.
+        """
+        session = DeclaredInputSession(session, self.in_channels)
+        channels = tuple(session.requested_output_channels or ())
+        missing = set(channels) - self.out_channels
+        if not channels or missing:
+            raise ValueError(f'Undeclared output channels {sorted(missing)}; declared: '
+                             f'{sorted(self.out_channels)}')
+        context = session.task_context
+        settings = measurement_settings(session)
+        if 'legacy_text' not in settings:
+            if channels == ('behavior',):
+                _drive_behavior_session_via_process(self, session)
+            else:
+                _drive_neural_session_via_process(self, session)
+            return
+        if context is not None and context.task_type != 'passive':
+            self.start_task(context)
+        for channel in channels:
+            if channel.startswith('neural:'):
+                self.start_neural_recording(channel.split(':', 1)[1],
+                                            settings.get('recording_type', 'fMRI'))
+        _drain_stream_events(session)
+        result = self._legacy.digest_text(settings['legacy_text'])
+        for channel in channels:
+            key = 'neural' if channel.startswith('neural:') else 'behavior'
+            session.respond(StreamEvent(channel, result[key], None, meta={'driver': 'legacy'}))
 
     # Backwards-compatible: existing benchmarks call digest_text() directly
     def digest_text(self, text):
         return self._legacy.digest_text(text)
 
     def start_task(self, task_context: TaskContext) -> None:
         self._task_context = task_context
-        self._task_active = True
         # Legacy ArtificialSubject.start_behavioral_task(task) takes ONE arg
-        self._legacy.start_behavioral_task(task_context.task_type)
+        self.start_behavioral_task(task_context.task_type)
 
     # Backwards-compatible: existing benchmarks call these directly
     def start_behavioral_task(self, task):
         self._task_active = True
+        if task == self._configured_task:
+            return
         self._legacy.start_behavioral_task(task)
+        self._configured_task = task
 
     def start_neural_recording(self, recording_target, recording_type='fMRI'):
         self._recording_active = True
+        registration = (recording_target, recording_type)
+        if registration in self._registered_recordings:
+            return  # legacy subjects append; the benchmark registered this once
         self._legacy.start_neural_recording(recording_target, recording_type)
+        self._registered_recordings.add(registration)
 
     def start_recording(self, recording_target: str,
                         time_bins=None, recording_type=None, **kwargs) -> None:
-        self._recording_active = True
         # Legacy ArtificialSubject.start_neural_recording(target, recording_type)
         # Default recording_type for language is 'fMRI'
-        self._legacy.start_neural_recording(
-            recording_target, recording_type or 'fMRI'
-        )
+        self.start_neural_recording(recording_target, recording_type or 'fMRI')
 
     def reset(self) -> None:
         reset = getattr(self._legacy, 'reset', None)
@@ -175,9 +253,16 @@ def reset(self) -> None:
         self._task_context = None
         self._recording_active = False
         self._task_active = False
+        self._registered_recordings = set()
+        self._configured_task = None
         if hasattr(self._legacy, 'current_tokens'):
             self._legacy.current_tokens = None
 
-    def __getattr__(self, name):
-        # Delegate attribute access to the legacy model for backwards compatibility
-        return getattr(self._legacy, name)
+    @property
+    def legacy_model(self):
+        """The wrapped legacy model, for tools that need its own attributes.
+
+        Unknown attributes are no longer forwarded implicitly: a benchmark
+        sees only what the adapter declares (Contract Rules v3, rule 2.1).
+        """
+        return self._legacy

requirements-v2.txt

@@ -1,2 +1,2 @@
 # Unpublished v2 core, pinned for standalone CI and documentation builds.
-brainscore-core @ git+https://github.com/brain-score/core.git@434eb311fca090567653716ce1cb337c35646b1b
+brainscore-core @ git+https://github.com/brain-score/core.git@0fbb419ff0ba461e6b1e217c845e0fc61d8096a1

tests/test_legacy_score.py

@@ -13,7 +13,8 @@ class LegacyBenchmark:
     parent = 'behavior'
 
     def __call__(self, model):
-        assert model.required_modalities == {'text'}
+        # The benchmark receives the legacy facade; the subject is behind it.
+        assert model.subject.required_channels == {"text"}
         return Score(0.5)
 
 

tests/test_preflight.py

@@ -45,7 +45,7 @@ def test_incompatible_model_raises_compatibility_error(
         mock_load_model.return_value = model
         mock_load_benchmark.return_value = benchmark
 
-        with pytest.raises(CompatibilityError, match="does not support modalities required"):
+        with pytest.raises(CompatibilityError, match="takes in none of the inputs"):
             _run_score('test-model', 'test-bench', check_mem=False)
 
     @patch('brainscore_language.load_benchmark')
@@ -95,6 +95,7 @@ def test_score_metadata_attrs_are_stamped(
         assert score.attrs['out_channels'] == ('behavior',)
         assert score.attrs['protocol'] == 'behavior:readout'
         assert score.attrs['subject_id'] == 'test-model'
+        assert score.attrs['compatibility']['match'] == 'full'
         assert score.attrs['harness_id'] == 'brainscore_language'
 
     @patch('brainscore_core.compatibility.check_channel_compatibility')

tests/test_submission/test_endpoints.py

@@ -85,4 +85,6 @@ def test_model_cannot_run(self):
         assert score_entry.score_raw is None
         assert score_entry.error is None
         assert 'NotImplementedError' in score_entry.comment
-        assert 'embedding.py' in score_entry.comment  # test for some stacktrace
+        assert "declares no 'behavior' output" in score_entry.comment
+        assert 'Traceback (most recent call last)' in score_entry.comment
+        assert 'legacy_facade.py' in score_entry.comment