fix(perf): stabilize task reference execution

benchmarks/performance/baselines/task_reference.py

@@ -244,6 +244,49 @@ def _snapshot_revision(path: str | Path) -> str:
     return ""
 
 
+def _load_sam3_processor(
+    transformers: Any,
+    model: str,
+    processor_kwargs: Mapping[str, Any],
+) -> Any:
+    """Load SAM3's processor, including Meta's processor_config-only snapshot."""
+    try:
+        return transformers.Sam3Processor.from_pretrained(model, **processor_kwargs)
+    except (OSError, ValueError) as processor_error:
+        model_path = Path(model)
+        if model_path.is_dir():
+            processor_config_path = model_path / "processor_config.json"
+        else:
+            try:
+                from huggingface_hub import try_to_load_from_cache
+
+                cached_path = try_to_load_from_cache(
+                    model,
+                    "processor_config.json",
+                    revision=str(processor_kwargs.get("revision") or "main"),
+                )
+            except Exception:  # noqa: BLE001 - preserve the original processor error
+                raise processor_error
+            processor_config_path = Path(cached_path) if isinstance(cached_path, str) else Path()
+        if not processor_config_path.is_file():
+            raise processor_error
+
+        processor_config = json.loads(processor_config_path.read_text(encoding="utf-8"))
+        image_processor_kwargs = processor_config.get("image_processor")
+        if not isinstance(image_processor_kwargs, dict):
+            raise processor_error
+        image_processor_kwargs = dict(image_processor_kwargs)
+        image_processor_kwargs.pop("image_processor_type", None)
+        image_processor_kwargs.pop("processor_class", None)
+        image_processor = transformers.Sam3ImageProcessorFast(**image_processor_kwargs)
+        tokenizer = transformers.AutoTokenizer.from_pretrained(model, **processor_kwargs)
+        return transformers.Sam3Processor(
+            image_processor,
+            tokenizer,
+            target_size=processor_config.get("target_size"),
+        )
+
+
 def _cached_snapshot_revision(repo_id: str, requested: str | None) -> str:
     try:
         from huggingface_hub import scan_cache_dir
@@ -479,6 +522,41 @@ def invoke() -> Mapping[str, Any]:
     return Session(invoke, revision, framework, reference_dependencies=dependencies)
 
 
+def _load_nemo_asr_reference_model(
+    arguments: argparse.Namespace,
+    *,
+    device: Any,
+) -> Any:
+    if (
+        arguments.family == "nemotron_speech_streaming"
+        and "nemotron-3.5-asr-streaming" in arguments.model.lower()
+    ):
+        from tools.reference.speech import load_nemotron35_asr_model
+
+        return load_nemotron35_asr_model(
+            model=arguments.model,
+            revision=arguments.revision or "",
+            local_files_only=arguments.local_files_only,
+            device=str(device),
+        )
+
+    import nemo.collections.asr as nemo_asr
+
+    return nemo_asr.models.ASRModel.from_pretrained(
+        arguments.model,
+        map_location="cpu",
+    )
+
+
+def _disable_nemo_asr_cuda_graphs(model: Any) -> bool:
+    """Disable NeMo's optional RNNT CUDA Graph decoder when available."""
+    decoding = getattr(getattr(model, "decoding", None), "decoding", None)
+    disable_cuda_graphs = getattr(decoding, "disable_cuda_graphs", None)
+    if not callable(disable_cuda_graphs):
+        return False
+    return bool(disable_cuda_graphs())
+
+
 def _load_asr(
     arguments: argparse.Namespace,
     request: Mapping[str, Any],
@@ -494,14 +572,10 @@ def _load_asr(
     device = torch.device("cuda")
 
     if arguments.family in {"canary", "nemotron_speech_streaming"}:
-        import nemo.collections.asr as nemo_asr
         from tools.validation.engine import _transcription_text
 
-        model = (
-            nemo_asr.models.ASRModel.from_pretrained(arguments.model, map_location="cpu")
-            .eval()
-            .to(device)
-        )
+        model = _load_nemo_asr_reference_model(arguments, device=device).eval().to(device)
+        _disable_nemo_asr_cuda_graphs(model)
         reference_dtype = _torch_dtype(torch, arguments.precision)
         autocast_dtype = reference_dtype if arguments.precision != "fp32" else torch.float16
         temporary = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
@@ -1722,7 +1796,7 @@ def invoke() -> Mapping[str, Any]:
             return {"mask_count": int(masks.shape[0]), **_tensor_summary(masks)}
 
     elif arguments.family == "sam3":
-        processor = transformers.Sam3Processor.from_pretrained(arguments.model, **processor_kwargs)
+        processor = _load_sam3_processor(transformers, arguments.model, processor_kwargs)
         model = transformers.Sam3Model.from_pretrained(arguments.model, **kwargs).eval().to(device)
         inputs = _to_device(
             processor(

tests/tools/test_perf_matrix.py

@@ -2079,6 +2079,78 @@ def reject_profile(*_args):
         perf_matrix._candidate_build_python_profile({"model": {"family": "example"}})
 
 
+def test_candidate_preflight_requires_modelopt_for_auto_fp8(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    calls = []
+    monkeypatch.setattr(
+        perf_matrix,
+        "_run_command",
+        lambda argv, _environment, timeout: calls.append((argv, timeout))
+        or {
+            "exit_code": 1,
+            "stderr_tail": "ModuleNotFoundError: No module named 'modelopt'",
+        },
+    )
+
+    with pytest.raises(
+        perf_matrix.PerfMatrixError,
+        match="candidate build Python profile 'base' is missing nvidia-modelopt",
+    ):
+        perf_matrix._candidate_build_dependency_preflight(
+            {
+                "model": {
+                    "build": {
+                        "quantization": {
+                            "format": "fp8",
+                            "scale_source": "modelopt",
+                        }
+                    }
+                }
+            },
+            profile="base",
+            python="/profile/python",
+            timeout_seconds=30,
+        )
+
+    assert calls == [
+        (
+            [
+                "/profile/python",
+                "-c",
+                "import modelopt.torch.quantization",
+            ],
+            30,
+        )
+    ]
+
+
+def test_candidate_preflight_skips_modelopt_for_non_calibrated_models(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(
+        perf_matrix,
+        "_run_command",
+        lambda *_args, **_kwargs: pytest.fail("dependency probe must not run"),
+    )
+
+    perf_matrix._candidate_build_dependency_preflight(
+        {
+            "model": {
+                "build": {
+                    "quantization": {
+                        "format": "fp8",
+                        "scale_source": "precomputed",
+                    }
+                }
+            }
+        },
+        profile="base",
+        python="/profile/python",
+        timeout_seconds=30,
+    )
+
+
 def test_seq2seq_token_framing_is_explicit_and_exact() -> None:
     runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/hf_transformers.py"))
     normalize = runner["_normalize_seq2seq_tokens"]
@@ -2261,6 +2333,60 @@ def fake_environment():
     }
 
 
+def test_nemotron35_perf_reference_uses_archive_compatible_loader(monkeypatch) -> None:
+    runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/task_reference.py"))
+    from tools.reference import speech
+
+    expected = object()
+    captured: dict[str, object] = {}
+
+    def load_compatible(**kwargs):
+        captured.update(kwargs)
+        return expected
+
+    monkeypatch.setattr(
+        speech,
+        "load_nemotron35_asr_model",
+        load_compatible,
+        raising=False,
+    )
+    arguments = Namespace(
+        family="nemotron_speech_streaming",
+        model="nvidia/nemotron-3.5-asr-streaming-0.6b",
+        revision="model-revision",
+        local_files_only=True,
+    )
+
+    model = runner["_load_nemo_asr_reference_model"](
+        arguments,
+        device="cuda",
+    )
+
+    assert model is expected
+    assert captured == {
+        "model": "nvidia/nemotron-3.5-asr-streaming-0.6b",
+        "revision": "model-revision",
+        "local_files_only": True,
+        "device": "cuda",
+    }
+
+
+def test_nemo_asr_perf_reference_disables_rnnt_cuda_graphs() -> None:
+    runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/task_reference.py"))
+    calls = 0
+
+    class GreedyDecoder:
+        def disable_cuda_graphs(self) -> bool:
+            nonlocal calls
+            calls += 1
+            return True
+
+    model = Namespace(decoding=Namespace(decoding=GreedyDecoder()))
+
+    assert runner["_disable_nemo_asr_cuda_graphs"](model) is True
+    assert calls == 1
+
+
 def test_vlm_adapter_routes_non_generic_families_to_owned_loaders() -> None:
     runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/task_reference.py"))
     deepseek = object()
@@ -2362,6 +2488,67 @@ def __call__(self, **_kwargs):
     assert session.invoke() == {"num_masks": 2, "height": 4, "width": 6}
 
 
+def test_sam3_reference_falls_back_to_processor_config(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/task_reference.py"))
+    snapshot = tmp_path / "snapshot"
+    snapshot.mkdir()
+    processor_config = snapshot / "processor_config.json"
+    processor_config.write_text(
+        json.dumps(
+            {
+                "target_size": 1008,
+                "image_processor": {
+                    "image_processor_type": "Sam3ImageProcessorFast",
+                    "size": {"height": 1008, "width": 1008},
+                },
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    class FakeProcessor:
+        def __init__(self, image_processor, tokenizer, *, target_size):
+            self.image_processor = image_processor
+            self.tokenizer = tokenizer
+            self.target_size = target_size
+
+        @classmethod
+        def from_pretrained(cls, *_args, **_kwargs):
+            raise OSError("preprocessor_config.json is absent")
+
+    class FakeImageProcessor:
+        def __init__(self, **kwargs):
+            self.kwargs = kwargs
+
+    class FakeTokenizer:
+        @classmethod
+        def from_pretrained(cls, *_args, **_kwargs):
+            return "tokenizer"
+
+    fake_transformers = Namespace(
+        AutoTokenizer=FakeTokenizer,
+        Sam3ImageProcessorFast=FakeImageProcessor,
+        Sam3Processor=FakeProcessor,
+    )
+    monkeypatch.setitem(
+        sys.modules,
+        "huggingface_hub",
+        Namespace(try_to_load_from_cache=lambda *_args, **_kwargs: str(processor_config)),
+    )
+
+    processor = runner["_load_sam3_processor"](
+        fake_transformers,
+        "facebook/sam3",
+        {"local_files_only": True, "revision": "pinned"},
+    )
+
+    assert processor.target_size == 1008
+    assert processor.tokenizer == "tokenizer"
+    assert processor.image_processor.kwargs == {"size": {"height": 1008, "width": 1008}}
+
+
 def test_locateanything_fallback_tokenizer_supports_batch_decode(
     tmp_path: Path, monkeypatch
 ) -> None:

tests/tools/test_trtmc_reference.py

@@ -108,6 +108,64 @@ def fake_pipeline(task: str, **kwargs):
     }
 
 
+def test_tts_transcriber_limits_cuda_memory(monkeypatch) -> None:
+    load_calls: list[dict[str, object]] = []
+    transcribe_calls: list[dict[str, object]] = []
+    loaded_processor = SimpleNamespace(
+        tokenizer=object(),
+        feature_extractor=SimpleNamespace(sampling_rate=16000),
+    )
+
+    class FakeTranscriber:
+        feature_extractor = SimpleNamespace(sampling_rate=16000)
+
+        def __call__(self, _waveforms, **kwargs):
+            transcribe_calls.append(kwargs)
+            return [{"text": "hello"}, {"text": "world"}]
+
+    class FakeModelLoader:
+        @staticmethod
+        def from_pretrained(_model_id: str, **kwargs):
+            load_calls.append(kwargs)
+            return object()
+
+    monkeypatch.setitem(
+        sys.modules,
+        "torch",
+        SimpleNamespace(
+            cuda=SimpleNamespace(is_available=lambda: True),
+            float16="fp16",
+        ),
+    )
+    monkeypatch.setitem(
+        sys.modules,
+        "transformers",
+        SimpleNamespace(
+            AutoModelForSpeechSeq2Seq=FakeModelLoader,
+            AutoProcessor=SimpleNamespace(
+                from_pretrained=lambda *_args, **_kwargs: loaded_processor
+            ),
+            pipeline=lambda *_args, **_kwargs: FakeTranscriber(),
+        ),
+    )
+    monkeypatch.setattr(speech, "_read_wav_float32", lambda _path: ([0.0], 16000))
+    monkeypatch.setattr(
+        speech,
+        "_resample_audio",
+        lambda audio, _source_rate, _target_rate: audio,
+    )
+
+    result = speech._transcribe_tts(
+        SimpleNamespace(device="cuda", local_files_only=True),
+        [Path("one.wav"), Path("two.wav")],
+        "openai/whisper-large-v3-turbo",
+    )
+
+    assert result == ["hello", "world"]
+    assert load_calls == [{"local_files_only": True, "torch_dtype": "fp16"}]
+    assert transcribe_calls == [{"batch_size": 1}]
+
+
 def _prepare_work(path: Path, *, model_manifest: str = "") -> None:
     path.mkdir(parents=True)
     (path / "answers.json").write_text(