From 8685feeee7e51fc4d78b8ba76a436fd938a67d14 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 13 Aug 2026 16:20:25 +0000 Subject: [PATCH 1/5] fix(perf): load Nemotron 3.5 archive compatibly --- .../performance/baselines/task_reference.py | 33 +++++++++++++--- tests/tools/test_perf_matrix.py | 38 +++++++++++++++++++ tools/reference/speech.py | 19 ++++++++++ 3 files changed, 84 insertions(+), 6 deletions(-) diff --git a/benchmarks/performance/baselines/task_reference.py b/benchmarks/performance/baselines/task_reference.py index 0928f91324..43c32e3f4e 100644 --- a/benchmarks/performance/baselines/task_reference.py +++ b/benchmarks/performance/baselines/task_reference.py @@ -479,6 +479,32 @@ def invoke() -> Mapping[str, Any]: return Session(invoke, revision, framework, reference_dependencies=dependencies) +def _load_nemo_asr_reference_model( + arguments: argparse.Namespace, + *, + device: Any, +) -> Any: + if ( + arguments.family == "nemotron_speech_streaming" + and "nemotron-3.5-asr-streaming" in arguments.model.lower() + ): + from tools.reference.speech import load_nemotron35_asr_model + + return load_nemotron35_asr_model( + model=arguments.model, + revision=arguments.revision or "", + local_files_only=arguments.local_files_only, + device=str(device), + ) + + import nemo.collections.asr as nemo_asr + + return nemo_asr.models.ASRModel.from_pretrained( + arguments.model, + map_location="cpu", + ) + + def _load_asr( arguments: argparse.Namespace, request: Mapping[str, Any], @@ -494,14 +520,9 @@ def _load_asr( device = torch.device("cuda") if arguments.family in {"canary", "nemotron_speech_streaming"}: - import nemo.collections.asr as nemo_asr from tools.validation.engine import _transcription_text - model = ( - nemo_asr.models.ASRModel.from_pretrained(arguments.model, map_location="cpu") - .eval() - .to(device) - ) + model = _load_nemo_asr_reference_model(arguments, device=device).eval().to(device) reference_dtype = _torch_dtype(torch, arguments.precision) autocast_dtype = reference_dtype if arguments.precision != "fp32" else torch.float16 temporary = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) diff --git a/tests/tools/test_perf_matrix.py b/tests/tools/test_perf_matrix.py index 79b8e86967..f3748d97ef 100644 --- a/tests/tools/test_perf_matrix.py +++ b/tests/tools/test_perf_matrix.py @@ -2261,6 +2261,44 @@ def fake_environment(): } +def test_nemotron35_perf_reference_uses_archive_compatible_loader(monkeypatch) -> None: + runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/task_reference.py")) + from tools.reference import speech + + expected = object() + captured: dict[str, object] = {} + + def load_compatible(**kwargs): + captured.update(kwargs) + return expected + + monkeypatch.setattr( + speech, + "load_nemotron35_asr_model", + load_compatible, + raising=False, + ) + arguments = Namespace( + family="nemotron_speech_streaming", + model="nvidia/nemotron-3.5-asr-streaming-0.6b", + revision="model-revision", + local_files_only=True, + ) + + model = runner["_load_nemo_asr_reference_model"]( + arguments, + device="cuda", + ) + + assert model is expected + assert captured == { + "model": "nvidia/nemotron-3.5-asr-streaming-0.6b", + "revision": "model-revision", + "local_files_only": True, + "device": "cuda", + } + + def test_vlm_adapter_routes_non_generic_families_to_owned_loaders() -> None: runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/task_reference.py")) deepseek = object() diff --git a/tools/reference/speech.py b/tools/reference/speech.py index 18ada922ed..c58a64b941 100644 --- a/tools/reference/speech.py +++ b/tools/reference/speech.py @@ -466,6 +466,25 @@ def _load_nemotron35_model( return torch, model +def load_nemotron35_asr_model( + *, + model: str, + device: str, + revision: str = "", + local_files_only: bool = False, +) -> Any: + """Load Nemotron 3.5 through its archive-compatible NeMo model class.""" + arguments = argparse.Namespace( + model=model, + model_revision=revision, + local_files_only=local_files_only, + device=device, + ) + archive = _resolve_nemotron35_archive(arguments) + _torch, loaded = _load_nemotron35_model(arguments, archive) + return loaded + + def _load_nemo_asr_model(arguments: argparse.Namespace, nemo_asr: Any) -> Any: if arguments.local_files_only: archive = _resolve_nemo_archive( From 731e5054959bc50e7b4e168c183dc70f1515bcc1 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 13 Aug 2026 17:02:18 +0000 Subject: [PATCH 2/5] fix(perf): preflight ModelOpt calibration dependency --- tests/tools/test_perf_matrix.py | 72 +++++++++++++++++++++++++++++++++ tools/perf_matrix.py | 38 ++++++++++++++++- 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/tests/tools/test_perf_matrix.py b/tests/tools/test_perf_matrix.py index f3748d97ef..47643f8c57 100644 --- a/tests/tools/test_perf_matrix.py +++ b/tests/tools/test_perf_matrix.py @@ -2079,6 +2079,78 @@ def reject_profile(*_args): perf_matrix._candidate_build_python_profile({"model": {"family": "example"}}) +def test_candidate_preflight_requires_modelopt_for_auto_fp8( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls = [] + monkeypatch.setattr( + perf_matrix, + "_run_command", + lambda argv, _environment, timeout: calls.append((argv, timeout)) + or { + "exit_code": 1, + "stderr_tail": "ModuleNotFoundError: No module named 'modelopt'", + }, + ) + + with pytest.raises( + perf_matrix.PerfMatrixError, + match="candidate build Python profile 'base' is missing nvidia-modelopt", + ): + perf_matrix._candidate_build_dependency_preflight( + { + "model": { + "build": { + "quantization": { + "format": "fp8", + "scale_source": "modelopt", + } + } + } + }, + profile="base", + python="/profile/python", + timeout_seconds=30, + ) + + assert calls == [ + ( + [ + "/profile/python", + "-c", + "import modelopt.torch.quantization", + ], + 30, + ) + ] + + +def test_candidate_preflight_skips_modelopt_for_non_calibrated_models( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + perf_matrix, + "_run_command", + lambda *_args, **_kwargs: pytest.fail("dependency probe must not run"), + ) + + perf_matrix._candidate_build_dependency_preflight( + { + "model": { + "build": { + "quantization": { + "format": "fp8", + "scale_source": "precomputed", + } + } + } + }, + profile="base", + python="/profile/python", + timeout_seconds=30, + ) + + def test_seq2seq_token_framing_is_explicit_and_exact() -> None: runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/hf_transformers.py")) normalize = runner["_normalize_seq2seq_tokens"] diff --git a/tools/perf_matrix.py b/tools/perf_matrix.py index e97e9f3ee2..5927de1fef 100644 --- a/tools/perf_matrix.py +++ b/tools/perf_matrix.py @@ -1179,6 +1179,36 @@ def _candidate_build_python_profile(resolved: Mapping[str, Any]) -> tuple[str, s return profile, python +def _candidate_build_dependency_preflight( + resolved: Mapping[str, Any], + *, + profile: str, + python: str, + timeout_seconds: int, +) -> None: + model = resolved.get("model", {}) + build = model.get("build", {}) if isinstance(model, Mapping) else {} + quantization = build.get("quantization", {}) if isinstance(build, Mapping) else {} + if not isinstance(quantization, Mapping) or ( + str(quantization.get("format", "")).strip().lower(), + str(quantization.get("scale_source", "")).strip().lower(), + ) != ("fp8", "modelopt"): + return + + command = _run_command( + [python, "-c", "import modelopt.torch.quantization"], + _command_environment(), + timeout_seconds, + ) + if command["exit_code"] != 0: + detail = str(command.get("stderr_tail", "")).strip() + suffix = f": {detail}" if detail else "" + raise PerfMatrixError( + f"candidate build Python profile {profile!r} is missing " + f"nvidia-modelopt required for automatic FP8 calibration{suffix}" + ) + + def _command_environment() -> dict[str, str]: environment = dict(os.environ) existing = environment.get("PYTHONPATH", "") @@ -1238,7 +1268,13 @@ def _resolve_candidate( raise PerfMatrixError( f"case {case['id']} includes asset loading but resolves no timed asset path" ) - profile, _ = _candidate_build_python_profile(value) + profile, python = _candidate_build_python_profile(value) + _candidate_build_dependency_preflight( + value, + profile=profile, + python=python, + timeout_seconds=options.timeout_seconds, + ) value["_candidate_build_python_profile"] = profile command.pop("stdout", None) return value, argv, command From 14ff33657703de04b918c0007f7cdb71aa8c906e Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 13 Aug 2026 18:55:29 +0000 Subject: [PATCH 3/5] fix(perf): load SAM3 processor config --- .../performance/baselines/task_reference.py | 45 +++++++++++++- tests/tools/test_perf_matrix.py | 61 +++++++++++++++++++ 2 files changed, 105 insertions(+), 1 deletion(-) diff --git a/benchmarks/performance/baselines/task_reference.py b/benchmarks/performance/baselines/task_reference.py index 43c32e3f4e..d0f004294f 100644 --- a/benchmarks/performance/baselines/task_reference.py +++ b/benchmarks/performance/baselines/task_reference.py @@ -244,6 +244,49 @@ def _snapshot_revision(path: str | Path) -> str: return "" +def _load_sam3_processor( + transformers: Any, + model: str, + processor_kwargs: Mapping[str, Any], +) -> Any: + """Load SAM3's processor, including Meta's processor_config-only snapshot.""" + try: + return transformers.Sam3Processor.from_pretrained(model, **processor_kwargs) + except (OSError, ValueError) as processor_error: + model_path = Path(model) + if model_path.is_dir(): + processor_config_path = model_path / "processor_config.json" + else: + try: + from huggingface_hub import try_to_load_from_cache + + cached_path = try_to_load_from_cache( + model, + "processor_config.json", + revision=str(processor_kwargs.get("revision") or "main"), + ) + except Exception: # noqa: BLE001 - preserve the original processor error + raise processor_error + processor_config_path = Path(cached_path) if isinstance(cached_path, str) else Path() + if not processor_config_path.is_file(): + raise processor_error + + processor_config = json.loads(processor_config_path.read_text(encoding="utf-8")) + image_processor_kwargs = processor_config.get("image_processor") + if not isinstance(image_processor_kwargs, dict): + raise processor_error + image_processor_kwargs = dict(image_processor_kwargs) + image_processor_kwargs.pop("image_processor_type", None) + image_processor_kwargs.pop("processor_class", None) + image_processor = transformers.Sam3ImageProcessorFast(**image_processor_kwargs) + tokenizer = transformers.AutoTokenizer.from_pretrained(model, **processor_kwargs) + return transformers.Sam3Processor( + image_processor, + tokenizer, + target_size=processor_config.get("target_size"), + ) + + def _cached_snapshot_revision(repo_id: str, requested: str | None) -> str: try: from huggingface_hub import scan_cache_dir @@ -1743,7 +1786,7 @@ def invoke() -> Mapping[str, Any]: return {"mask_count": int(masks.shape[0]), **_tensor_summary(masks)} elif arguments.family == "sam3": - processor = transformers.Sam3Processor.from_pretrained(arguments.model, **processor_kwargs) + processor = _load_sam3_processor(transformers, arguments.model, processor_kwargs) model = transformers.Sam3Model.from_pretrained(arguments.model, **kwargs).eval().to(device) inputs = _to_device( processor( diff --git a/tests/tools/test_perf_matrix.py b/tests/tools/test_perf_matrix.py index 47643f8c57..55722b3030 100644 --- a/tests/tools/test_perf_matrix.py +++ b/tests/tools/test_perf_matrix.py @@ -2472,6 +2472,67 @@ def __call__(self, **_kwargs): assert session.invoke() == {"num_masks": 2, "height": 4, "width": 6} +def test_sam3_reference_falls_back_to_processor_config( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/task_reference.py")) + snapshot = tmp_path / "snapshot" + snapshot.mkdir() + processor_config = snapshot / "processor_config.json" + processor_config.write_text( + json.dumps( + { + "target_size": 1008, + "image_processor": { + "image_processor_type": "Sam3ImageProcessorFast", + "size": {"height": 1008, "width": 1008}, + }, + } + ), + encoding="utf-8", + ) + + class FakeProcessor: + def __init__(self, image_processor, tokenizer, *, target_size): + self.image_processor = image_processor + self.tokenizer = tokenizer + self.target_size = target_size + + @classmethod + def from_pretrained(cls, *_args, **_kwargs): + raise OSError("preprocessor_config.json is absent") + + class FakeImageProcessor: + def __init__(self, **kwargs): + self.kwargs = kwargs + + class FakeTokenizer: + @classmethod + def from_pretrained(cls, *_args, **_kwargs): + return "tokenizer" + + fake_transformers = Namespace( + AutoTokenizer=FakeTokenizer, + Sam3ImageProcessorFast=FakeImageProcessor, + Sam3Processor=FakeProcessor, + ) + monkeypatch.setitem( + sys.modules, + "huggingface_hub", + Namespace(try_to_load_from_cache=lambda *_args, **_kwargs: str(processor_config)), + ) + + processor = runner["_load_sam3_processor"]( + fake_transformers, + "facebook/sam3", + {"local_files_only": True, "revision": "pinned"}, + ) + + assert processor.target_size == 1008 + assert processor.tokenizer == "tokenizer" + assert processor.image_processor.kwargs == {"size": {"height": 1008, "width": 1008}} + + def test_locateanything_fallback_tokenizer_supports_batch_decode( tmp_path: Path, monkeypatch ) -> None: From 0ac656855ffdab21b8978e039e00e636d9fb7c13 Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 13 Aug 2026 19:01:12 +0000 Subject: [PATCH 4/5] fix(validation): bound TTS transcription memory --- tests/tools/test_trtmc_reference.py | 58 +++++++++++++++++++++++++++++ tools/reference/speech.py | 16 ++++---- 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/tests/tools/test_trtmc_reference.py b/tests/tools/test_trtmc_reference.py index ffe4cb43b3..3f895dad70 100644 --- a/tests/tools/test_trtmc_reference.py +++ b/tests/tools/test_trtmc_reference.py @@ -108,6 +108,64 @@ def fake_pipeline(task: str, **kwargs): } +def test_tts_transcriber_limits_cuda_memory(monkeypatch) -> None: + load_calls: list[dict[str, object]] = [] + transcribe_calls: list[dict[str, object]] = [] + loaded_processor = SimpleNamespace( + tokenizer=object(), + feature_extractor=SimpleNamespace(sampling_rate=16000), + ) + + class FakeTranscriber: + feature_extractor = SimpleNamespace(sampling_rate=16000) + + def __call__(self, _waveforms, **kwargs): + transcribe_calls.append(kwargs) + return [{"text": "hello"}, {"text": "world"}] + + class FakeModelLoader: + @staticmethod + def from_pretrained(_model_id: str, **kwargs): + load_calls.append(kwargs) + return object() + + monkeypatch.setitem( + sys.modules, + "torch", + SimpleNamespace( + cuda=SimpleNamespace(is_available=lambda: True), + float16="fp16", + ), + ) + monkeypatch.setitem( + sys.modules, + "transformers", + SimpleNamespace( + AutoModelForSpeechSeq2Seq=FakeModelLoader, + AutoProcessor=SimpleNamespace( + from_pretrained=lambda *_args, **_kwargs: loaded_processor + ), + pipeline=lambda *_args, **_kwargs: FakeTranscriber(), + ), + ) + monkeypatch.setattr(speech, "_read_wav_float32", lambda _path: ([0.0], 16000)) + monkeypatch.setattr( + speech, + "_resample_audio", + lambda audio, _source_rate, _target_rate: audio, + ) + + result = speech._transcribe_tts( + SimpleNamespace(device="cuda", local_files_only=True), + [Path("one.wav"), Path("two.wav")], + "openai/whisper-large-v3-turbo", + ) + + assert result == ["hello", "world"] + assert load_calls == [{"local_files_only": True, "torch_dtype": "fp16"}] + assert transcribe_calls == [{"batch_size": 1}] + + def _prepare_work(path: Path, *, model_manifest: str = "") -> None: path.mkdir(parents=True) (path / "answers.json").write_text( diff --git a/tools/reference/speech.py b/tools/reference/speech.py index c58a64b941..d85b6194bb 100644 --- a/tools/reference/speech.py +++ b/tools/reference/speech.py @@ -690,14 +690,16 @@ def _transcribe_tts( pipeline, ) - device = ( - 0 - if arguments.device.startswith("cuda") and torch.cuda.is_available() - else -1 - ) + use_cuda = arguments.device.startswith("cuda") and torch.cuda.is_available() + device = 0 if use_cuda else -1 + model_kwargs: dict[str, Any] = { + "local_files_only": arguments.local_files_only, + } + if use_cuda: + model_kwargs["torch_dtype"] = torch.float16 model = AutoModelForSpeechSeq2Seq.from_pretrained( model_id, - local_files_only=arguments.local_files_only, + **model_kwargs, ) processor = AutoProcessor.from_pretrained( model_id, @@ -715,7 +717,7 @@ def _transcribe_tts( for path in wav_paths: audio, sample_rate = _read_wav_float32(str(path)) waveforms.append(_resample_audio(audio, sample_rate, target_rate)) - outputs = transcriber(waveforms, batch_size=min(8, len(waveforms))) + outputs = transcriber(waveforms, batch_size=1 if use_cuda else min(8, len(waveforms))) if isinstance(outputs, Mapping): outputs = [outputs] return [_transcription_text(output).strip() for output in outputs] From 2e5feb369baa208bc5cf52883321e84b92e432bd Mon Sep 17 00:00:00 2001 From: chaofengw Date: Thu, 13 Aug 2026 20:24:29 +0000 Subject: [PATCH 5/5] fix(perf): disable NeMo RNNT CUDA graphs --- .../performance/baselines/task_reference.py | 10 ++++++++++ tests/tools/test_perf_matrix.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/benchmarks/performance/baselines/task_reference.py b/benchmarks/performance/baselines/task_reference.py index d0f004294f..572e195060 100644 --- a/benchmarks/performance/baselines/task_reference.py +++ b/benchmarks/performance/baselines/task_reference.py @@ -548,6 +548,15 @@ def _load_nemo_asr_reference_model( ) +def _disable_nemo_asr_cuda_graphs(model: Any) -> bool: + """Disable NeMo's optional RNNT CUDA Graph decoder when available.""" + decoding = getattr(getattr(model, "decoding", None), "decoding", None) + disable_cuda_graphs = getattr(decoding, "disable_cuda_graphs", None) + if not callable(disable_cuda_graphs): + return False + return bool(disable_cuda_graphs()) + + def _load_asr( arguments: argparse.Namespace, request: Mapping[str, Any], @@ -566,6 +575,7 @@ def _load_asr( from tools.validation.engine import _transcription_text model = _load_nemo_asr_reference_model(arguments, device=device).eval().to(device) + _disable_nemo_asr_cuda_graphs(model) reference_dtype = _torch_dtype(torch, arguments.precision) autocast_dtype = reference_dtype if arguments.precision != "fp32" else torch.float16 temporary = tempfile.NamedTemporaryFile(suffix=".wav", delete=False) diff --git a/tests/tools/test_perf_matrix.py b/tests/tools/test_perf_matrix.py index 55722b3030..79d02bd6d3 100644 --- a/tests/tools/test_perf_matrix.py +++ b/tests/tools/test_perf_matrix.py @@ -2371,6 +2371,22 @@ def load_compatible(**kwargs): } +def test_nemo_asr_perf_reference_disables_rnnt_cuda_graphs() -> None: + runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/task_reference.py")) + calls = 0 + + class GreedyDecoder: + def disable_cuda_graphs(self) -> bool: + nonlocal calls + calls += 1 + return True + + model = Namespace(decoding=Namespace(decoding=GreedyDecoder())) + + assert runner["_disable_nemo_asr_cuda_graphs"](model) is True + assert calls == 1 + + def test_vlm_adapter_routes_non_generic_families_to_owned_loaders() -> None: runner = runpy.run_path(str(REPOSITORY / "benchmarks/performance/baselines/task_reference.py")) deepseek = object()