From 462a982823a3fb1e10e4fb5edbaf5a3bc8e1267a Mon Sep 17 00:00:00 2001 From: yifeif-nv Date: Fri, 28 Aug 2026 11:16:41 -0700 Subject: [PATCH] ci: frontload all CPU unit tests Run the complete CPU-safe C++ and Python suite for every pull request while keeping GPU model proofs selective. Label GPU-only tests, isolate CPU mocks, and add JSON/TOML contracts so model-owned serialization regressions cannot escape premerge. Signed-off-by: yifeif-nv --- .github/workflows/community-cpu.yml | 31 +-- CMakeLists.txt | 33 ++- Dockerfile.community-cpu | 1 + .../minimax_h3/tests/test_trt_builders.py | 13 + .../families/sana_wm/tests/test_family.py | 13 +- requirements/community-ci.txt | 1 + src/runtime/models/albert/MODEL.toml | 2 +- src/runtime/models/bark/MODEL.toml | 2 +- src/runtime/models/bert/MODEL.toml | 2 +- src/runtime/models/canary/MODEL.toml | 2 +- src/runtime/models/convbert/MODEL.toml | 2 +- src/runtime/models/deberta/MODEL.toml | 2 +- src/runtime/models/deepseek_ocr/MODEL.toml | 2 +- src/runtime/models/distilbert/MODEL.toml | 2 +- src/runtime/models/dpr/MODEL.toml | 2 +- src/runtime/models/eagle_vlm/MODEL.toml | 2 +- src/runtime/models/electra/MODEL.toml | 2 +- src/runtime/models/elf_flow/MODEL.toml | 2 +- src/runtime/models/flux/MODEL.toml | 3 +- src/runtime/models/fnet/MODEL.toml | 2 +- src/runtime/models/internlm/MODEL.toml | 6 +- src/runtime/models/lance/MODEL.toml | 2 +- src/runtime/models/llama/MODEL.toml | 8 +- src/runtime/models/magpie/MODEL.toml | 2 +- src/runtime/models/mamba/MODEL.toml | 8 +- src/runtime/models/modernbert/MODEL.toml | 2 +- src/runtime/models/mpnet/MODEL.toml | 2 +- src/runtime/models/nemotron_h/MODEL.toml | 8 +- .../models/nemotron_labs_diffusion/MODEL.toml | 6 +- src/runtime/models/personaplex/MODEL.toml | 4 +- src/runtime/models/phi4_multimodal/MODEL.toml | 2 +- src/runtime/models/qwen/MODEL.toml | 2 +- src/runtime/models/qwen3_omni/MODEL.toml | 2 +- src/runtime/models/roberta/MODEL.toml | 2 +- src/runtime/models/rwkv/MODEL.toml | 8 +- src/runtime/models/sam/MODEL.toml | 2 +- src/runtime/models/sam3/MODEL.toml | 2 +- src/runtime/models/segformer/MODEL.toml | 2 +- src/runtime/models/starcoder2/MODEL.toml | 6 +- src/runtime/models/wan/MODEL.toml | 2 +- src/runtime/models/xlnet/MODEL.toml | 2 +- .../models/flux/test_flux_host_contracts.cpp | 98 ++++++++ tests/cpp/models/flux/test_flux_pipeline.cpp | 70 ------ tests/cpp/test_json_helpers.cpp | 26 +- .../test_albert_build_engine_integration.py | 2 + .../test_bart_build_engine_integration.py | 2 + .../test_codegen_build_engine_integration.py | 2 + .../test_deberta_build_engine_integration.py | 2 + .../test_deepseek_ocr_builder_tp.py | 2 + .../dinov3/test_dinov3_convnext_builder.py | 4 + .../models/dinov3/test_dinov3_vit_builder.py | 6 + ...est_distilbert_build_engine_integration.py | 2 + .../dpr/test_dpr_build_engine_integration.py | 2 + .../eagle_vlm/test_eagle_vlm_builder_tp.py | 4 + .../test_electra_build_engine_integration.py | 2 + .../test_fnet_build_engine_integration.py | 2 + .../test_gpt2_build_engine_integration.py | 2 + .../test_gpt_neo_build_engine_integration.py | 2 + .../test_gpt_neox_build_engine_integration.py | 2 + .../gpt_oss/test_gpt_oss_rope_and_sliding.py | 2 + .../test_internlm_build_engine_integration.py | 2 + .../test_m2m_100_build_engine_integration.py | 2 + ...est_modernbert_build_engine_integration.py | 2 + .../test_olmo2_build_engine_integration.py | 2 + .../test_phi4_multimodal_family_plugin.py | 2 + tests/e2e/models/qwen/test_perf_parity.py | 2 + .../t5/test_t5_build_engine_integration.py | 2 + tests/tools/test_community_ci.py | 32 ++- tests/tools/test_github_actions_ci.py | 124 +++++++++- tests/tools/test_model_ci.py | 188 ++++++++++++++- .../test_model_plugin_encapsulation_static.py | 51 ++++ tools/ci/README.md | 26 +- tools/ci/quality.py | 144 +++++++++-- tools/model_ci.py | 228 +++++++++++++++++- website/docs/extend/contributing.md | 17 +- 75 files changed, 1025 insertions(+), 234 deletions(-) create mode 100644 tests/cpp/models/flux/test_flux_host_contracts.cpp diff --git a/.github/workflows/community-cpu.yml b/.github/workflows/community-cpu.yml index 21c576d3ef..dae4672f73 100644 --- a/.github/workflows/community-cpu.yml +++ b/.github/workflows/community-cpu.yml @@ -110,31 +110,12 @@ jobs: unit: name: Community CPU / Unit / C++ and Python - if: ${{ !cancelled() }} - needs: ownership-impact runs-on: ubuntu-24.04 timeout-minutes: 45 permissions: contents: read steps: - - name: Require successful impact analysis - if: ${{ needs.ownership-impact.result != 'success' }} - env: - IMPACT_RESULT: ${{ needs.ownership-impact.result }} - run: | - echo "::error::Ownership and impact did not succeed: $IMPACT_RESULT" - exit 1 - - - name: Explain an empty unit scope - if: >- - needs.ownership-impact.result == 'success' && - needs.ownership-impact.outputs.run_unit_tests != 'true' - run: echo "No source-only C++ or Python unit tests are required for this change." - - name: Check out the exact PR merge - if: >- - needs.ownership-impact.result == 'success' && - needs.ownership-impact.outputs.run_unit_tests == 'true' uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: ref: ${{ github.sha }} @@ -142,22 +123,12 @@ jobs: persist-credentials: false - name: Set up Python - if: >- - needs.ownership-impact.result == 'success' && - needs.ownership-impact.outputs.run_unit_tests == 'true' uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 with: python-version: "3.12" - name: Run hardened source-only units - if: >- - needs.ownership-impact.result == 'success' && - needs.ownership-impact.outputs.run_unit_tests == 'true' - env: - TRTMC_UNIT_SCOPE: ${{ needs.ownership-impact.outputs.unit_scope }} - TRTMC_PREMERGE_PYTHON_TEST_TARGETS: >- - ${{ needs.ownership-impact.outputs.python_test_targets }} - run: python3 -m tools.community_ci unit --scope "$TRTMC_UNIT_SCOPE" + run: python3 -m tools.community_ci unit --scope all required: name: Community CPU / Required diff --git a/CMakeLists.txt b/CMakeLists.txt index 8dfbab7732..3f5d7ec199 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -722,6 +722,7 @@ if(TRTMC_BUILD_TESTS) enable_testing() add_custom_target(trtmc_cpp_tests) add_custom_target(trtmc_platform_cpp_tests) +add_custom_target(trtmc_cpu_cpp_tests) # Generic capsule test doubles. The good and wrong-identity DSOs intentionally # have the same filename in different directories so tests can prove that the @@ -784,8 +785,8 @@ set_target_properties(trtmc_test_optimized_embedding PROPERTIES # Usage: # trtmc_add_test(test_name) -- links trtmc_core, adds src/ include # trtmc_add_test(test_name REQUIRES_TRT) -- also links TensorRT test deps -# trtmc_add_test(test_name MODEL_OWNED) -- excludes it from platform units -# trtmc_add_test(test_name REQUIRES_GPU) -- excludes it from CPU platform units +# trtmc_add_test(test_name MODEL_OWNED) -- labels model ownership independently +# trtmc_add_test(test_name REQUIRES_GPU) -- excludes it from all CPU unit targets function(trtmc_add_test TEST_NAME) cmake_parse_arguments( ARG @@ -794,6 +795,10 @@ function(trtmc_add_test TEST_NAME) "EXTRA_INCLUDES" ${ARGN} ) + if(ARG_UNPARSED_ARGUMENTS) + message(FATAL_ERROR + "Unknown trtmc_add_test option(s) for '${TEST_NAME}': ${ARG_UNPARSED_ARGUMENTS}") + endif() if(ARG_REQUIRES_TRT AND NOT _trtmc_can_build_trt_backend) message(STATUS "Skipping ${TEST_NAME}: TensorRT SDK not available") return() @@ -830,13 +835,23 @@ function(trtmc_add_test TEST_NAME) endif() add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) if(ARG_MODEL_OWNED) - set_tests_properties(${TEST_NAME} PROPERTIES LABELS "model") - elseif(ARG_REQUIRES_GPU) - set_tests_properties(${TEST_NAME} PROPERTIES LABELS "gpu") + set(_trtmc_test_owner_label "model") + else() + set(_trtmc_test_owner_label "platform") + endif() + if(ARG_REQUIRES_GPU) + set(_trtmc_test_resource_label "gpu") else() + set(_trtmc_test_resource_label "cpu") + add_dependencies(trtmc_cpu_cpp_tests ${TEST_NAME}) + endif() + if(NOT ARG_MODEL_OWNED AND NOT ARG_REQUIRES_GPU) add_dependencies(trtmc_platform_cpp_tests ${TEST_NAME}) - set_tests_properties(${TEST_NAME} PROPERTIES LABELS "platform") endif() + set_tests_properties( + ${TEST_NAME} + PROPERTIES LABELS "${_trtmc_test_owner_label};${_trtmc_test_resource_label}" + ) endfunction() function(_trtmc_manifest_csv_to_list VALUE OUTPUT_VAR) @@ -931,7 +946,11 @@ if(_trtmc_python) ) add_custom_target(test_optimized_runtime_bundle_contract DEPENDS test_optimized_runtime_host trtmc_test_optimized_runtime) - set_tests_properties(test_optimized_runtime_bundle_contract PROPERTIES LABELS "platform") + add_dependencies(trtmc_cpu_cpp_tests test_optimized_runtime_bundle_contract) + set_tests_properties( + test_optimized_runtime_bundle_contract + PROPERTIES LABELS "platform;cpu" + ) endif() # C ABI trtmc_add_test(test_c_abi_entry NO_SRC_INCLUDE) diff --git a/Dockerfile.community-cpu b/Dockerfile.community-cpu index b4a41af587..ca782c52b7 100644 --- a/Dockerfile.community-cpu +++ b/Dockerfile.community-cpu @@ -30,6 +30,7 @@ RUN apt-get update && \ python3.12-dev \ python3.12-venv \ libcublas-dev-13-3 \ + libcurand-dev-13-3 \ "libnvinfer-dev=${TENSORRT_APT_VERSION}" \ "libnvinfer-headers-dev=${TENSORRT_APT_VERSION}" \ "libnvinfer-headers-plugin-dev=${TENSORRT_APT_VERSION}" \ diff --git a/python/tensorrt_model_connect/families/minimax_h3/tests/test_trt_builders.py b/python/tensorrt_model_connect/families/minimax_h3/tests/test_trt_builders.py index de7625d6da..0bf564a953 100644 --- a/python/tensorrt_model_connect/families/minimax_h3/tests/test_trt_builders.py +++ b/python/tensorrt_model_connect/families/minimax_h3/tests/test_trt_builders.py @@ -130,6 +130,17 @@ def capture(_config, supplied, *, default_bytes): observed.update(supplied=supplied, default_bytes=default_bytes) raise WorkspaceConfigured + class FakeBuilder: + @staticmethod + def create_network(_flags): + return object() + + @staticmethod + def create_builder_config(): + return object() + + monkeypatch.setattr(trt, "Builder", lambda _logger: FakeBuilder()) + monkeypatch.setattr(op, "configure_builder", lambda _config: None) monkeypatch.setattr(op, "configure_workspace", capture) kwargs = {"workspace_bytes": workspace_bytes} if builder is build_text_encoder_engine: @@ -303,6 +314,8 @@ def test_native_linear_serializes_checkpoint_bf16_without_fp32_constant() -> Non assert plan +@pytest.mark.gpu +@pytest.mark.trt def test_native_network_contract_counts_iattention_and_fails_closed() -> None: logger = trt.Logger(trt.Logger.WARNING) builder = trt.Builder(logger) diff --git a/python/tensorrt_model_connect/families/sana_wm/tests/test_family.py b/python/tensorrt_model_connect/families/sana_wm/tests/test_family.py index 2712aae39e..10b0703909 100644 --- a/python/tensorrt_model_connect/families/sana_wm/tests/test_family.py +++ b/python/tensorrt_model_connect/families/sana_wm/tests/test_family.py @@ -42,7 +42,7 @@ @pytest.fixture(autouse=True) -def _clear_sana_wm_env(monkeypatch) -> None: +def _clear_sana_wm_env(monkeypatch, tmp_path) -> None: for name in _SANA_WM_ENV_VARS: monkeypatch.delenv(name, raising=False) monkeypatch.setattr( @@ -50,6 +50,17 @@ def _clear_sana_wm_env(monkeypatch) -> None: "_cached_stage1_text_encoder_dir", lambda _raw_config: None, ) + native_plugin = tmp_path / "libtrtmc_sana_wm_native_plugin.so" + native_plugin.write_bytes(b"CPU unit native plugin fixture") + native_plugin_builder = importlib.import_module( + "tensorrt_model_connect.families.sana_wm.native_plugin_builder" + ) + monkeypatch.setattr( + native_plugin_builder, + "ensure_native_plugin", + lambda **_kwargs: native_plugin, + ) + monkeypatch.setattr(sana_wm_plugin_mod.ctypes, "CDLL", lambda *_args, **_kwargs: object()) def _sana_yaml() -> str: diff --git a/requirements/community-ci.txt b/requirements/community-ci.txt index 8f32bf2af2..78e1137da0 100644 --- a/requirements/community-ci.txt +++ b/requirements/community-ci.txt @@ -11,3 +11,4 @@ pytest==8.4.2 pytest-xdist==3.8.0 jsonschema==4.26.0 Pillow==12.2.0 +pyarrow==25.0.1 diff --git a/src/runtime/models/albert/MODEL.toml b/src/runtime/models/albert/MODEL.toml index abbf6c8ada..9a908f9f17 100644 --- a/src/runtime/models/albert/MODEL.toml +++ b/src/runtime/models/albert/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_albert.so" runtime_plugins = ["plugin.cpp|register_albert_plugin"] runtime_strategies = ["albert_encoder_only"] runtime_tests = [ - "test_albert_encoder_pipeline|test_albert_encoder_pipeline.cpp|trtmc_model_albert,trtmc_backend_trt|_|REQUIRES_TRT", + "test_albert_encoder_pipeline|test_albert_encoder_pipeline.cpp|trtmc_model_albert,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/bark/MODEL.toml b/src/runtime/models/bark/MODEL.toml index 2cde5662d3..36f47e50bb 100644 --- a/src/runtime/models/bark/MODEL.toml +++ b/src/runtime/models/bark/MODEL.toml @@ -10,5 +10,5 @@ legacy_runtime_strategy_aliases = ["text_to_audio|default|_|_|text_to_audio_bark runtime_tests = [ "test_bark_config_schema|test_bark_config_schema.cpp|trtmc_model_bark|_|_", "test_bark_generation_plan|test_bark_generation_plan.cpp|_|_|_", - "test_bark_pipeline|test_bark_pipeline.cpp|trtmc_model_bark|_|REQUIRES_TRT", + "test_bark_pipeline|test_bark_pipeline.cpp|trtmc_model_bark|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/bert/MODEL.toml b/src/runtime/models/bert/MODEL.toml index bd4304da7c..5de3860827 100644 --- a/src/runtime/models/bert/MODEL.toml +++ b/src/runtime/models/bert/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_bert.so" runtime_plugins = ["plugin.cpp|register_bert_plugin"] runtime_strategies = ["bert_encoder_only", "bert_embedding"] runtime_tests = [ - "test_bert_encoder_pipeline|test_bert_encoder_pipeline.cpp|trtmc_model_bert,trtmc_backend_trt|_|REQUIRES_TRT", + "test_bert_encoder_pipeline|test_bert_encoder_pipeline.cpp|trtmc_model_bert,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/canary/MODEL.toml b/src/runtime/models/canary/MODEL.toml index ab54ab5d80..f6dc411816 100644 --- a/src/runtime/models/canary/MODEL.toml +++ b/src/runtime/models/canary/MODEL.toml @@ -9,5 +9,5 @@ runtime_tests = [ "test_canary_mel_spectrogram|test_canary_mel_spectrogram.cpp|_|canary_mel_spectrogram.cpp|_", "test_canary_host_plan|test_canary_host_plan.cpp|_|_|_", "test_canary_decode_policy|test_canary_decode_policy.cpp|_|_|_", - "test_canary_pipeline|test_canary_pipeline.cpp|trtmc_model_canary|_|REQUIRES_TRT", + "test_canary_pipeline|test_canary_pipeline.cpp|trtmc_model_canary|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/convbert/MODEL.toml b/src/runtime/models/convbert/MODEL.toml index 77504ada04..2b822d2615 100644 --- a/src/runtime/models/convbert/MODEL.toml +++ b/src/runtime/models/convbert/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_convbert.so" runtime_plugins = ["plugin.cpp|register_convbert_plugin"] runtime_strategies = ["convbert_encoder_only"] runtime_tests = [ - "test_convbert_encoder_pipeline|test_convbert_encoder_pipeline.cpp|trtmc_model_convbert,trtmc_backend_trt|_|REQUIRES_TRT", + "test_convbert_encoder_pipeline|test_convbert_encoder_pipeline.cpp|trtmc_model_convbert,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/deberta/MODEL.toml b/src/runtime/models/deberta/MODEL.toml index 33a96b1681..4cfb444e1b 100644 --- a/src/runtime/models/deberta/MODEL.toml +++ b/src/runtime/models/deberta/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_deberta.so" runtime_plugins = ["plugin.cpp|register_deberta_plugin"] runtime_strategies = ["deberta_encoder_only"] runtime_tests = [ - "test_deberta_encoder_pipeline|test_deberta_encoder_pipeline.cpp|trtmc_model_deberta,trtmc_backend_trt|_|REQUIRES_TRT", + "test_deberta_encoder_pipeline|test_deberta_encoder_pipeline.cpp|trtmc_model_deberta,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/deepseek_ocr/MODEL.toml b/src/runtime/models/deepseek_ocr/MODEL.toml index 2638dd7328..9ce4287f5b 100644 --- a/src/runtime/models/deepseek_ocr/MODEL.toml +++ b/src/runtime/models/deepseek_ocr/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_deepseek_ocr.so" runtime_plugins = ["plugin.cpp|register_deepseek_ocr_plugin"] runtime_strategies = ["deepseek_ocr_vision_language"] runtime_tests = [ - "test_deepseek_ocr_vl_pipeline|test_deepseek_ocr_vl_pipeline.cpp|trtmc_model_deepseek_ocr,trtmc_backend_trt|_|REQUIRES_TRT", + "test_deepseek_ocr_vl_pipeline|test_deepseek_ocr_vl_pipeline.cpp|trtmc_model_deepseek_ocr,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/distilbert/MODEL.toml b/src/runtime/models/distilbert/MODEL.toml index 80a0be48bc..d382d90b0f 100644 --- a/src/runtime/models/distilbert/MODEL.toml +++ b/src/runtime/models/distilbert/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_distilbert.so" runtime_plugins = ["plugin.cpp|register_distilbert_plugin"] runtime_strategies = ["distilbert_encoder_only"] runtime_tests = [ - "test_distilbert_encoder_pipeline|test_distilbert_encoder_pipeline.cpp|trtmc_model_distilbert,trtmc_backend_trt|_|REQUIRES_TRT", + "test_distilbert_encoder_pipeline|test_distilbert_encoder_pipeline.cpp|trtmc_model_distilbert,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/dpr/MODEL.toml b/src/runtime/models/dpr/MODEL.toml index b7d66e13b3..6fc07f0b68 100644 --- a/src/runtime/models/dpr/MODEL.toml +++ b/src/runtime/models/dpr/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_dpr.so" runtime_plugins = ["plugin.cpp|register_dpr_plugin"] runtime_strategies = ["dpr_encoder_only"] runtime_tests = [ - "test_dpr_encoder_pipeline|test_dpr_encoder_pipeline.cpp|trtmc_model_dpr,trtmc_backend_trt|_|REQUIRES_TRT", + "test_dpr_encoder_pipeline|test_dpr_encoder_pipeline.cpp|trtmc_model_dpr,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/eagle_vlm/MODEL.toml b/src/runtime/models/eagle_vlm/MODEL.toml index 1738dede5b..2aba9d7b9b 100644 --- a/src/runtime/models/eagle_vlm/MODEL.toml +++ b/src/runtime/models/eagle_vlm/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_eagle_vlm.so" runtime_plugins = ["plugin.cpp|register_eagle_vlm_plugin"] runtime_strategies = ["eagle_vlm_embedding", "eagle_vlm_reranking"] runtime_tests = [ - "test_eagle_vlm_encoder_pipeline|test_eagle_vlm_encoder_pipeline.cpp|trtmc_model_eagle_vlm,trtmc_backend_trt|_|REQUIRES_TRT", + "test_eagle_vlm_encoder_pipeline|test_eagle_vlm_encoder_pipeline.cpp|trtmc_model_eagle_vlm,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/electra/MODEL.toml b/src/runtime/models/electra/MODEL.toml index b64a6a9c7a..8cd53b9de0 100644 --- a/src/runtime/models/electra/MODEL.toml +++ b/src/runtime/models/electra/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_electra.so" runtime_plugins = ["plugin.cpp|register_electra_plugin"] runtime_strategies = ["electra_encoder_only"] runtime_tests = [ - "test_electra_encoder_pipeline|test_electra_encoder_pipeline.cpp|trtmc_model_electra,trtmc_backend_trt|_|REQUIRES_TRT", + "test_electra_encoder_pipeline|test_electra_encoder_pipeline.cpp|trtmc_model_electra,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/elf_flow/MODEL.toml b/src/runtime/models/elf_flow/MODEL.toml index 7d48805f89..2b832f1492 100644 --- a/src/runtime/models/elf_flow/MODEL.toml +++ b/src/runtime/models/elf_flow/MODEL.toml @@ -7,6 +7,6 @@ runtime_plugins = ["plugin.cpp|register_elf_flow_plugin"] runtime_strategies = ["elf_flow"] runtime_tests = [ "test_elf_flow_pipeline|test_elf_flow_pipeline.cpp|trtmc_model_elf_flow|_|_", - "test_elf_flow_sampling_kernels|test_elf_flow_sampling_kernels.cpp|trtmc_model_elf_flow|_|REQUIRES_TRT", + "test_elf_flow_sampling_kernels|test_elf_flow_sampling_kernels.cpp|trtmc_model_elf_flow|_|REQUIRES_TRT,REQUIRES_GPU", ] task_strategy = "diffusion_text_generation" diff --git a/src/runtime/models/flux/MODEL.toml b/src/runtime/models/flux/MODEL.toml index 13ff5e4206..8714015f05 100644 --- a/src/runtime/models/flux/MODEL.toml +++ b/src/runtime/models/flux/MODEL.toml @@ -8,7 +8,8 @@ runtime_strategies = ["diffusion_flux"] legacy_runtime_strategy_aliases = ["diffusion|contains|diffusion_backend_type|flux|diffusion_flux"] runtime_link_libraries = ["cublas"] runtime_tests = [ - "test_flux_pipeline|test_flux_pipeline.cpp|trtmc_model_flux|_|_", + "test_flux_host_contracts|test_flux_host_contracts.cpp|_|_|_", + "test_flux_pipeline|test_flux_pipeline.cpp|trtmc_model_flux|_|REQUIRES_GPU", "test_flux_generation_plan|test_flux_generation_plan.cpp|_|_|_", "test_flux_denoising_step_seam|test_flux_denoising_step_seam.cpp|_|_|_", "test_flux_batch_utils|test_flux_batch_utils.cpp|_|_|_", diff --git a/src/runtime/models/fnet/MODEL.toml b/src/runtime/models/fnet/MODEL.toml index 7a30755b84..bd8c6b033b 100644 --- a/src/runtime/models/fnet/MODEL.toml +++ b/src/runtime/models/fnet/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_fnet.so" runtime_plugins = ["plugin.cpp|register_fnet_plugin"] runtime_strategies = ["fnet_encoder_only"] runtime_tests = [ - "test_fnet_encoder_pipeline|test_fnet_encoder_pipeline.cpp|trtmc_model_fnet,trtmc_backend_trt|_|REQUIRES_TRT", + "test_fnet_encoder_pipeline|test_fnet_encoder_pipeline.cpp|trtmc_model_fnet,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/internlm/MODEL.toml b/src/runtime/models/internlm/MODEL.toml index ef52eb4e6d..186658387e 100644 --- a/src/runtime/models/internlm/MODEL.toml +++ b/src/runtime/models/internlm/MODEL.toml @@ -5,10 +5,10 @@ id = "internlm" runtime_library = "libtrtmc_model_internlm.so" runtime_plugins = ["plugin.cpp|register_internlm_plugin"] runtime_strategies = ["internlm_decoder_kv_cache"] -[validation_profiles] -decoder_debug = ["internlm_decoder_kv_cache"] - runtime_tests = [ "test_internlm_chat_template|test_internlm_chat_template.cpp|trtmc_model_internlm|_|_", "test_internlm_pipeline|test_internlm_pipeline.cpp|trtmc_model_internlm|_|REQUIRES_GPU", ] + +[validation_profiles] +decoder_debug = ["internlm_decoder_kv_cache"] diff --git a/src/runtime/models/lance/MODEL.toml b/src/runtime/models/lance/MODEL.toml index a8820aaa38..d79bd9fae6 100644 --- a/src/runtime/models/lance/MODEL.toml +++ b/src/runtime/models/lance/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_lance.so" runtime_plugins = ["plugin.cpp|register_lance_plugin"] runtime_strategies = ["lance_vision_language"] runtime_tests = [ - "test_lance_vl_pipeline|test_lance_vl_pipeline.cpp|trtmc_model_lance,trtmc_backend_trt|_|REQUIRES_TRT", + "test_lance_vl_pipeline|test_lance_vl_pipeline.cpp|trtmc_model_lance,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/llama/MODEL.toml b/src/runtime/models/llama/MODEL.toml index eda9a40590..e748acc432 100644 --- a/src/runtime/models/llama/MODEL.toml +++ b/src/runtime/models/llama/MODEL.toml @@ -5,12 +5,12 @@ id = "llama" runtime_library = "libtrtmc_model_llama.so" runtime_plugins = ["plugin.cpp|register_llama_plugin"] runtime_strategies = ["llama_decoder_kv_cache"] -[validation_profiles] -decoder_debug = ["llama_decoder_kv_cache"] - runtime_tests = [ "test_llama_plugin_helpers|test_llama_plugin_helpers.cpp|_|plugin_helpers.cpp|_", "test_llama_chat_template|test_llama_chat_template.cpp|trtmc_model_llama|_|_", - "test_llama_pipeline|test_llama_pipeline.cpp|trtmc_model_llama,trtmc_backend_trt|_|REQUIRES_TRT", + "test_llama_pipeline|test_llama_pipeline.cpp|trtmc_model_llama,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", "test_llama_native_kv_cache|test_llama_native_kv_cache.cpp|trtmc_model_llama|_|REQUIRES_GPU", ] + +[validation_profiles] +decoder_debug = ["llama_decoder_kv_cache"] diff --git a/src/runtime/models/magpie/MODEL.toml b/src/runtime/models/magpie/MODEL.toml index 473bffd2e4..a914ef7eb4 100644 --- a/src/runtime/models/magpie/MODEL.toml +++ b/src/runtime/models/magpie/MODEL.toml @@ -15,5 +15,5 @@ runtime_tests = [ "test_magpie_generation_plan|test_magpie_generation_plan.cpp|_|_|_", "test_magpie_text_completion_policy|test_magpie_text_completion_policy.cpp|_|_|_", "test_magpie_codec_plan|test_magpie_codec_plan.cpp|_|_|_", - "test_magpie_pipeline|test_magpie_pipeline.cpp|trtmc_model_magpie|_|REQUIRES_TRT", + "test_magpie_pipeline|test_magpie_pipeline.cpp|trtmc_model_magpie|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/mamba/MODEL.toml b/src/runtime/models/mamba/MODEL.toml index e183e553e5..59bbfb7157 100644 --- a/src/runtime/models/mamba/MODEL.toml +++ b/src/runtime/models/mamba/MODEL.toml @@ -5,10 +5,10 @@ id = "mamba" runtime_library = "libtrtmc_model_mamba.so" runtime_plugins = ["plugin.cpp|register_mamba_plugin"] runtime_strategies = ["mamba_ssm_recurrent"] -[validation_profiles] -decoder_debug = ["mamba_ssm_recurrent"] - runtime_tests = [ "test_mamba_recurrent_output_initializers|test_mamba_recurrent_output_initializers.cpp|_|_|_", - "test_mamba_recurrent_pipeline|test_mamba_recurrent_pipeline.cpp|trtmc_model_mamba,trtmc_backend_trt|_|REQUIRES_TRT", + "test_mamba_recurrent_pipeline|test_mamba_recurrent_pipeline.cpp|trtmc_model_mamba,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] + +[validation_profiles] +decoder_debug = ["mamba_ssm_recurrent"] diff --git a/src/runtime/models/modernbert/MODEL.toml b/src/runtime/models/modernbert/MODEL.toml index 033aa0ae02..e309d03bfa 100644 --- a/src/runtime/models/modernbert/MODEL.toml +++ b/src/runtime/models/modernbert/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_modernbert.so" runtime_plugins = ["plugin.cpp|register_modernbert_plugin"] runtime_strategies = ["modernbert_encoder_only"] runtime_tests = [ - "test_modernbert_encoder_pipeline|test_modernbert_encoder_pipeline.cpp|trtmc_model_modernbert,trtmc_backend_trt|_|REQUIRES_TRT", + "test_modernbert_encoder_pipeline|test_modernbert_encoder_pipeline.cpp|trtmc_model_modernbert,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/mpnet/MODEL.toml b/src/runtime/models/mpnet/MODEL.toml index a87fb62124..f7c02ebcf2 100644 --- a/src/runtime/models/mpnet/MODEL.toml +++ b/src/runtime/models/mpnet/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_mpnet.so" runtime_plugins = ["plugin.cpp|register_mpnet_plugin"] runtime_strategies = ["mpnet_encoder_only"] runtime_tests = [ - "test_mpnet_encoder_pipeline|test_mpnet_encoder_pipeline.cpp|trtmc_model_mpnet,trtmc_backend_trt|_|REQUIRES_TRT", + "test_mpnet_encoder_pipeline|test_mpnet_encoder_pipeline.cpp|trtmc_model_mpnet,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/nemotron_h/MODEL.toml b/src/runtime/models/nemotron_h/MODEL.toml index c7f7df0609..4212a13468 100644 --- a/src/runtime/models/nemotron_h/MODEL.toml +++ b/src/runtime/models/nemotron_h/MODEL.toml @@ -5,11 +5,11 @@ id = "nemotron_h" runtime_library = "libtrtmc_model_nemotron_h.so" runtime_plugins = ["plugin.cpp|register_nemotron_h_plugin"] runtime_strategies = ["nemotron_h_hybrid_mamba_attention"] -[validation_profiles] -decoder_debug = ["nemotron_h_hybrid_mamba_attention"] - runtime_tests = [ "test_nemotron_h_backend_cuda_graph|test_nemotron_h_backend_cuda_graph.cpp|trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", "test_nemotron_h_recurrent_output_initializers|test_nemotron_h_recurrent_output_initializers.cpp|_|_|_", - "test_nemotron_h_recurrent_pipeline|test_nemotron_h_recurrent_pipeline.cpp|trtmc_model_nemotron_h,trtmc_backend_trt|_|REQUIRES_TRT", + "test_nemotron_h_recurrent_pipeline|test_nemotron_h_recurrent_pipeline.cpp|trtmc_model_nemotron_h,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] + +[validation_profiles] +decoder_debug = ["nemotron_h_hybrid_mamba_attention"] diff --git a/src/runtime/models/nemotron_labs_diffusion/MODEL.toml b/src/runtime/models/nemotron_labs_diffusion/MODEL.toml index a3a7f1e6f0..3cb6a3537b 100644 --- a/src/runtime/models/nemotron_labs_diffusion/MODEL.toml +++ b/src/runtime/models/nemotron_labs_diffusion/MODEL.toml @@ -5,9 +5,9 @@ id = "nemotron_labs_diffusion" runtime_library = "libtrtmc_model_nemotron_labs_diffusion.so" runtime_plugins = ["plugin.cpp|register_nemotron_labs_diffusion_plugin"] runtime_strategies = ["nemotron_labs_diffusion"] -[validation_profiles] -decoder_debug = ["nemotron_labs_diffusion"] - runtime_tests = [ "test_nemotron_labs_diffusion_chat_template|test_nemotron_labs_diffusion_chat_template.cpp|trtmc_model_nemotron_labs_diffusion|_|_", ] + +[validation_profiles] +decoder_debug = ["nemotron_labs_diffusion"] diff --git a/src/runtime/models/personaplex/MODEL.toml b/src/runtime/models/personaplex/MODEL.toml index 3aa5fb9c54..879bd9f399 100644 --- a/src/runtime/models/personaplex/MODEL.toml +++ b/src/runtime/models/personaplex/MODEL.toml @@ -16,6 +16,6 @@ runtime_tests = [ "test_personaplex_speech_temporal_embed_plan|test_personaplex_speech_temporal_embed_plan.cpp|_|_|_", "test_personaplex_speech_mimi_decode_plan|test_personaplex_speech_mimi_decode_plan.cpp|_|_|_", "test_personaplex_decode_runtime|test_personaplex_decode_runtime.cpp|_|decode_runtime.cpp|_", - "test_personaplex_sampling_kernels|test_personaplex_sampling_kernels.cpp|trtmc_model_personaplex|_|REQUIRES_TRT", - "test_personaplex_speech_pipeline|test_personaplex_speech_pipeline.cpp|trtmc_model_personaplex|_|REQUIRES_TRT", + "test_personaplex_sampling_kernels|test_personaplex_sampling_kernels.cpp|trtmc_model_personaplex|_|REQUIRES_TRT,REQUIRES_GPU", + "test_personaplex_speech_pipeline|test_personaplex_speech_pipeline.cpp|trtmc_model_personaplex|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/phi4_multimodal/MODEL.toml b/src/runtime/models/phi4_multimodal/MODEL.toml index 46120722e7..2271691d80 100644 --- a/src/runtime/models/phi4_multimodal/MODEL.toml +++ b/src/runtime/models/phi4_multimodal/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_phi4_multimodal.so" runtime_plugins = ["plugin.cpp|register_phi4_multimodal_plugin"] runtime_strategies = ["phi4_multimodal_vision_language"] runtime_tests = [ - "test_phi4_multimodal_vl_pipeline|test_phi4_multimodal_vl_pipeline.cpp|trtmc_model_phi4_multimodal,trtmc_backend_trt|_|REQUIRES_TRT", + "test_phi4_multimodal_vl_pipeline|test_phi4_multimodal_vl_pipeline.cpp|trtmc_model_phi4_multimodal,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/qwen/MODEL.toml b/src/runtime/models/qwen/MODEL.toml index 8b891d54e5..922b111f6f 100644 --- a/src/runtime/models/qwen/MODEL.toml +++ b/src/runtime/models/qwen/MODEL.toml @@ -6,7 +6,7 @@ runtime_library = "libtrtmc_model_qwen.so" runtime_plugins = ["plugin.cpp|register_qwen_plugin"] runtime_strategies = ["qwen_decoder_kv_cache"] runtime_tests = [ - "test_c_abi_runtime_regression|test_c_abi_runtime_regression.cpp|trtmc_model_qwen|_|_", + "test_c_abi_runtime_regression|test_c_abi_runtime_regression.cpp|trtmc_model_qwen|_|REQUIRES_GPU", "test_qwen_tensor_names|test_qwen_tensor_names.cpp|_|_|_", "test_qwen_sampler|test_qwen_sampler.cpp|trtmc_model_qwen|_|_", "test_qwen_native_kv_cache_dtype|test_qwen_native_kv_cache_dtype.cpp|trtmc_model_qwen|_|_", diff --git a/src/runtime/models/qwen3_omni/MODEL.toml b/src/runtime/models/qwen3_omni/MODEL.toml index 9c316a68bc..d4685a94a2 100644 --- a/src/runtime/models/qwen3_omni/MODEL.toml +++ b/src/runtime/models/qwen3_omni/MODEL.toml @@ -8,5 +8,5 @@ runtime_strategies = ["qwen3_omni_multimodal"] runtime_tests = [ "test_qwen3_omni_audio_plan|test_qwen3_omni_audio_plan.cpp|_|_|_", "test_qwen3_omni_talker_runtime|test_qwen3_omni_talker_runtime.cpp|_|talker_runtime.cpp|_", - "test_qwen3_omni_pipeline|test_qwen3_omni_pipeline.cpp|trtmc_model_qwen3_omni|_|REQUIRES_TRT", + "test_qwen3_omni_pipeline|test_qwen3_omni_pipeline.cpp|trtmc_model_qwen3_omni|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/roberta/MODEL.toml b/src/runtime/models/roberta/MODEL.toml index 0cb2f8ec15..65ca648497 100644 --- a/src/runtime/models/roberta/MODEL.toml +++ b/src/runtime/models/roberta/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_roberta.so" runtime_plugins = ["plugin.cpp|register_roberta_plugin"] runtime_strategies = ["roberta_encoder_only"] runtime_tests = [ - "test_roberta_encoder_pipeline|test_roberta_encoder_pipeline.cpp|trtmc_model_roberta,trtmc_backend_trt|_|REQUIRES_TRT", + "test_roberta_encoder_pipeline|test_roberta_encoder_pipeline.cpp|trtmc_model_roberta,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/rwkv/MODEL.toml b/src/runtime/models/rwkv/MODEL.toml index 2103e99830..0890a637d1 100644 --- a/src/runtime/models/rwkv/MODEL.toml +++ b/src/runtime/models/rwkv/MODEL.toml @@ -5,10 +5,10 @@ id = "rwkv" runtime_library = "libtrtmc_model_rwkv.so" runtime_plugins = ["plugin.cpp|register_rwkv_plugin"] runtime_strategies = ["rwkv_recurrent"] -[validation_profiles] -decoder_debug = ["rwkv_recurrent"] - runtime_tests = [ "test_rwkv_recurrent_output_initializers|test_rwkv_recurrent_output_initializers.cpp|_|_|_", - "test_rwkv_recurrent_pipeline|test_rwkv_recurrent_pipeline.cpp|trtmc_model_rwkv,trtmc_backend_trt|_|REQUIRES_TRT", + "test_rwkv_recurrent_pipeline|test_rwkv_recurrent_pipeline.cpp|trtmc_model_rwkv,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] + +[validation_profiles] +decoder_debug = ["rwkv_recurrent"] diff --git a/src/runtime/models/sam/MODEL.toml b/src/runtime/models/sam/MODEL.toml index 8180ed9a27..af1da7a989 100644 --- a/src/runtime/models/sam/MODEL.toml +++ b/src/runtime/models/sam/MODEL.toml @@ -8,5 +8,5 @@ runtime_strategies = ["sam_prompted_segmentation"] runtime_tests = [ "test_sam_prompt_seam|test_sam_prompt_seam.cpp|trtmc_model_sam|_|_", "test_sam_image_preprocess_seam|test_sam_image_preprocess_seam.cpp|trtmc_model_sam|_|_", - "test_sam_pipeline|test_sam_pipeline.cpp|trtmc_model_sam|_|REQUIRES_TRT", + "test_sam_pipeline|test_sam_pipeline.cpp|trtmc_model_sam|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/sam3/MODEL.toml b/src/runtime/models/sam3/MODEL.toml index 786842500e..b2725d77a4 100644 --- a/src/runtime/models/sam3/MODEL.toml +++ b/src/runtime/models/sam3/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_sam3.so" runtime_plugins = ["plugin.cpp|register_sam3_plugin"] runtime_strategies = ["sam3_prompted_segmentation"] runtime_tests = [ - "test_sam3_pipeline|test_sam3_pipeline.cpp|trtmc_model_sam3|_|_", + "test_sam3_pipeline|test_sam3_pipeline.cpp|trtmc_model_sam3|_|REQUIRES_GPU", ] diff --git a/src/runtime/models/segformer/MODEL.toml b/src/runtime/models/segformer/MODEL.toml index 3e3897dcbd..3d623b2c06 100644 --- a/src/runtime/models/segformer/MODEL.toml +++ b/src/runtime/models/segformer/MODEL.toml @@ -8,5 +8,5 @@ runtime_strategies = ["segformer_segmentation"] runtime_tests = [ "test_segformer_preprocess_seam|test_segformer_preprocess_seam.cpp|trtmc_model_segformer|_|_", "test_segformer_postprocess_seam|test_segformer_postprocess_seam.cpp|_|_|_", - "test_segformer_segment_pipeline|test_segformer_segment_pipeline.cpp|trtmc_model_segformer|_|REQUIRES_TRT", + "test_segformer_segment_pipeline|test_segformer_segment_pipeline.cpp|trtmc_model_segformer|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/src/runtime/models/starcoder2/MODEL.toml b/src/runtime/models/starcoder2/MODEL.toml index 21760065a7..e5e184b2ed 100644 --- a/src/runtime/models/starcoder2/MODEL.toml +++ b/src/runtime/models/starcoder2/MODEL.toml @@ -5,9 +5,9 @@ id = "starcoder2" runtime_library = "libtrtmc_model_starcoder2.so" runtime_plugins = ["plugin.cpp|register_starcoder2_plugin"] runtime_strategies = ["starcoder2_decoder_kv_cache"] -[validation_profiles] -decoder_debug = ["starcoder2_decoder_kv_cache"] - runtime_tests = [ "test_starcoder2_tokenizer_contract|test_starcoder2_tokenizer_contract.cpp|_|_|_", ] + +[validation_profiles] +decoder_debug = ["starcoder2_decoder_kv_cache"] diff --git a/src/runtime/models/wan/MODEL.toml b/src/runtime/models/wan/MODEL.toml index c1fd360e64..6697814001 100644 --- a/src/runtime/models/wan/MODEL.toml +++ b/src/runtime/models/wan/MODEL.toml @@ -10,7 +10,7 @@ gnu_warning_suppressed_sources = ["pipeline.cpp"] runtime_link_libraries = ["cublas"] runtime_tests = [ "test_wan_pipeline|test_wan_pipeline.cpp|trtmc_model_wan|_|_", - "test_wan_c_abi_runtime_regression|test_wan_c_abi_runtime_regression.cpp|trtmc_model_wan|_|_", + "test_wan_c_abi_runtime_regression|test_wan_c_abi_runtime_regression.cpp|trtmc_model_wan|_|REQUIRES_GPU", "test_wan_generation_plan|test_wan_generation_plan.cpp|_|_|_", "test_wan_generation_conditioning|test_wan_generation_conditioning.cpp|_|_|_", "test_wan_denoising_step_seam|test_wan_denoising_step_seam.cpp|_|_|_", diff --git a/src/runtime/models/xlnet/MODEL.toml b/src/runtime/models/xlnet/MODEL.toml index 973549a3fb..4f536271ae 100644 --- a/src/runtime/models/xlnet/MODEL.toml +++ b/src/runtime/models/xlnet/MODEL.toml @@ -6,5 +6,5 @@ runtime_library = "libtrtmc_model_xlnet.so" runtime_plugins = ["plugin.cpp|register_xlnet_plugin"] runtime_strategies = ["xlnet_encoder_only"] runtime_tests = [ - "test_xlnet_encoder_pipeline|test_xlnet_encoder_pipeline.cpp|trtmc_model_xlnet,trtmc_backend_trt|_|REQUIRES_TRT", + "test_xlnet_encoder_pipeline|test_xlnet_encoder_pipeline.cpp|trtmc_model_xlnet,trtmc_backend_trt|_|REQUIRES_TRT,REQUIRES_GPU", ] diff --git a/tests/cpp/models/flux/test_flux_host_contracts.cpp b/tests/cpp/models/flux/test_flux_host_contracts.cpp new file mode 100644 index 0000000000..32a4b2640f --- /dev/null +++ b/tests/cpp/models/flux/test_flux_host_contracts.cpp @@ -0,0 +1,98 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "runtime/models/flux/flux_clip_helpers.h" +#include "runtime/models/flux/flux_rope_helpers.h" +#include "runtime/models/flux/flux_text_helpers.h" + +#include +#include +#include + +namespace { + +int failures = 0; + +class PadTokenizer final : public trtmc::ITokenizer { + public: + std::vector encode(const std::string&) const override { return {}; } + std::string decode(const std::vector&) const override { return {}; } + int32_t id_for_token(std::string_view token) const override { + return token == "" ? 11 : -1; + } + std::string token_for_id(int32_t) const override { return {}; } +}; + +void check(bool condition, const char* name) { + if (!condition) { + std::cerr << "FAIL: " << name << '\n'; + ++failures; + } +} + +void test_clip_padding_preserves_eos_when_truncated() { + using trtmc::diffusion::flux_clip::pad_and_truncate_ids; + + check(pad_and_truncate_ids({10, 1, 11}, 5, 11, 11) == std::vector({10, 1, 11, 11, 11}), + "CLIP short input pads with EOS"); + check(pad_and_truncate_ids({10, 1, 2, 3, 11}, 5, 11, 11) == + std::vector({10, 1, 2, 3, 11}), + "CLIP exact input preserves EOS"); + check(pad_and_truncate_ids({10, 1, 2, 3, 4, 11}, 5, 11, 11) == + std::vector({10, 1, 2, 3, 11}), + "CLIP truncated input restores EOS"); +} + +void test_flux2_text_padding_matches_tokenizer_contract() { + using trtmc::diffusion::flux_text::clear_padding_rows; + using trtmc::diffusion::flux_text::prepare_inputs; + using trtmc::diffusion::flux_text::resolve_pad_token_id; + + PadTokenizer tokenizer; + check(resolve_pad_token_id(&tokenizer, true) == 11, "FLUX.2 resolves tokenizer pad id"); + check(resolve_pad_token_id(&tokenizer, false) == 0, "FLUX.1 retains legacy pad id"); + + const auto prepared = prepare_inputs({12, 0, 13}, 5, 11); + check(prepared.input_ids == std::vector({12, 0, 13, 11, 11}), + "FLUX.2 text uses tokenizer pad id"); + check(prepared.attention_mask == std::vector({0.0F, 0.0F, 0.0F, -1e9F, -1e9F}), + "FLUX.2 text mask follows input length rather than token value"); + + std::vector embeddings(10, 1.0F); + clear_padding_rows(embeddings, {3}, 5, 2, true); + check(std::all_of(embeddings.begin(), embeddings.end(), + [](float value) { return value == 1.0F; }), + "FLUX.2 preserves padded query embeddings"); + clear_padding_rows(embeddings, {3}, 5, 2, false); + check(embeddings == + std::vector({1.0F, 1.0F, 1.0F, 1.0F, 1.0F, 1.0F, 0.0F, 0.0F, 0.0F, 0.0F}), + "FLUX.1 clears padded query embeddings"); +} + +void test_flux2_rope_coordinates_match_diffusers_contract() { + using trtmc::diffusion::flux_rope::axis_position; + + check(axis_position(0, 0, 0, 0, 17) == 0, "FLUX.2 text temporal coordinate"); + check(axis_position(1, 0, 0, 0, 17) == 0, "FLUX.2 text height coordinate"); + check(axis_position(2, 0, 0, 0, 17) == 0, "FLUX.2 text width coordinate"); + check(axis_position(3, 0, 0, 0, 17) == 17, "FLUX.2 text sequence coordinate"); + + check(axis_position(0, 0, 5, 7, 0) == 0, "FLUX.2 image temporal coordinate"); + check(axis_position(1, 0, 5, 7, 0) == 5, "FLUX.2 image height coordinate"); + check(axis_position(2, 0, 5, 7, 0) == 7, "FLUX.2 image width coordinate"); + check(axis_position(3, 0, 5, 7, 0) == 0, "FLUX.2 image sequence coordinate"); +} + +} // namespace + +int main() { + test_clip_padding_preserves_eos_when_truncated(); + test_flux2_text_padding_matches_tokenizer_contract(); + test_flux2_rope_coordinates_match_diffusers_contract(); + if (failures > 0) { + std::cerr << failures << " flux host contract test(s) FAILED\n"; + } + return failures; +} diff --git a/tests/cpp/models/flux/test_flux_pipeline.cpp b/tests/cpp/models/flux/test_flux_pipeline.cpp index 56e21bfc51..c9445e8c9f 100644 --- a/tests/cpp/models/flux/test_flux_pipeline.cpp +++ b/tests/cpp/models/flux/test_flux_pipeline.cpp @@ -3,12 +3,8 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "runtime/models/flux/flux_clip_helpers.h" -#include "runtime/models/flux/flux_rope_helpers.h" -#include "runtime/models/flux/flux_text_helpers.h" #include "runtime/models/flux/pipeline.h" -#include #include #include @@ -16,16 +12,6 @@ namespace { int failures = 0; -class PadTokenizer final : public trtmc::ITokenizer { - public: - std::vector encode(const std::string&) const override { return {}; } - std::string decode(const std::vector&) const override { return {}; } - int32_t id_for_token(std::string_view token) const override { - return token == "" ? 11 : -1; - } - std::string token_for_id(int32_t) const override { return {}; } -}; - void check(bool condition, const char* name) { if (!condition) { std::cerr << "FAIL: " << name << '\n'; @@ -57,67 +43,11 @@ void test_flux_with_custom_config() { "FluxPipeline custom config pipeline_type"); } -void test_clip_padding_preserves_eos_when_truncated() { - using trtmc::diffusion::flux_clip::pad_and_truncate_ids; - - check(pad_and_truncate_ids({10, 1, 11}, 5, 11, 11) == std::vector({10, 1, 11, 11, 11}), - "CLIP short input pads with EOS"); - check(pad_and_truncate_ids({10, 1, 2, 3, 11}, 5, 11, 11) == - std::vector({10, 1, 2, 3, 11}), - "CLIP exact input preserves EOS"); - check(pad_and_truncate_ids({10, 1, 2, 3, 4, 11}, 5, 11, 11) == - std::vector({10, 1, 2, 3, 11}), - "CLIP truncated input restores EOS"); -} - -void test_flux2_text_padding_matches_tokenizer_contract() { - using trtmc::diffusion::flux_text::clear_padding_rows; - using trtmc::diffusion::flux_text::prepare_inputs; - using trtmc::diffusion::flux_text::resolve_pad_token_id; - - PadTokenizer tokenizer; - check(resolve_pad_token_id(&tokenizer, true) == 11, "FLUX.2 resolves tokenizer pad id"); - check(resolve_pad_token_id(&tokenizer, false) == 0, "FLUX.1 retains legacy pad id"); - - const auto prepared = prepare_inputs({12, 0, 13}, 5, 11); - check(prepared.input_ids == std::vector({12, 0, 13, 11, 11}), - "FLUX.2 text uses tokenizer pad id"); - check(prepared.attention_mask == std::vector({0.0F, 0.0F, 0.0F, -1e9F, -1e9F}), - "FLUX.2 text mask follows input length rather than token value"); - - std::vector embeddings(10, 1.0F); - clear_padding_rows(embeddings, {3}, 5, 2, true); - check(std::all_of(embeddings.begin(), embeddings.end(), - [](float value) { return value == 1.0F; }), - "FLUX.2 preserves padded query embeddings"); - clear_padding_rows(embeddings, {3}, 5, 2, false); - check(embeddings == - std::vector({1.0F, 1.0F, 1.0F, 1.0F, 1.0F, 1.0F, 0.0F, 0.0F, 0.0F, 0.0F}), - "FLUX.1 clears padded query embeddings"); -} - -void test_flux2_rope_coordinates_match_diffusers_contract() { - using trtmc::diffusion::flux_rope::axis_position; - - check(axis_position(0, 0, 0, 0, 17) == 0, "FLUX.2 text temporal coordinate"); - check(axis_position(1, 0, 0, 0, 17) == 0, "FLUX.2 text height coordinate"); - check(axis_position(2, 0, 0, 0, 17) == 0, "FLUX.2 text width coordinate"); - check(axis_position(3, 0, 0, 0, 17) == 17, "FLUX.2 text sequence coordinate"); - - check(axis_position(0, 0, 5, 7, 0) == 0, "FLUX.2 image temporal coordinate"); - check(axis_position(1, 0, 5, 7, 0) == 5, "FLUX.2 image height coordinate"); - check(axis_position(2, 0, 5, 7, 0) == 7, "FLUX.2 image width coordinate"); - check(axis_position(3, 0, 5, 7, 0) == 0, "FLUX.2 image sequence coordinate"); -} - } // namespace int main() { test_flux_construction(); test_flux_with_custom_config(); - test_clip_padding_preserves_eos_when_truncated(); - test_flux2_text_padding_matches_tokenizer_contract(); - test_flux2_rope_coordinates_match_diffusers_contract(); if (failures > 0) { std::cerr << failures << " flux pipeline test(s) FAILED\n"; } diff --git a/tests/cpp/test_json_helpers.cpp b/tests/cpp/test_json_helpers.cpp index eff12808a9..850a382063 100644 --- a/tests/cpp/test_json_helpers.cpp +++ b/tests/cpp/test_json_helpers.cpp @@ -19,12 +19,10 @@ // ============================================================================= // // Purpose: -// Validates the lightweight, regex-free JSON extraction functions used -// throughout the codebase to parse HuggingFace config.json and -// generation_config.json files. These extractors operate on raw JSON strings -// (no DOM parser) and must correctly handle common JSON patterns including -// nested objects, arrays, negative integers, scientific-notation floats, -// and missing keys (fallback values). +// Validates the strict JSON extraction functions used throughout the +// codebase to parse HuggingFace config.json and generation_config.json. +// Ordinary field lookup is deliberately scoped to the top-level object; +// callers must explicitly extract an object before reading nested fields. // // Dependencies: // - utils/json_helpers.h (extract_json_string, extract_json_int, @@ -158,6 +156,20 @@ bool test_extract_json_int_missing() { return true; } +// Intention: Lock strict top-level lookup so nested model fields cannot be +// consumed accidentally as if they were bundle runtime fields. +bool test_extract_json_int_nested_only_uses_fallback() { + const std::string json = R"({"text_config": {"hidden_size": 1536}})"; + return trtmc::extract_json_int(json, "hidden_size", -99) == -99; +} + +// Intention: When a composite config contains both nested source metadata and +// a materialized runtime field, the top-level field is authoritative. +bool test_extract_json_int_top_level_wins_over_nested() { + const std::string json = R"({"text_config": {"hidden_size": 4096}, "hidden_size": 1536})"; + return trtmc::extract_json_int(json, "hidden_size", -99) == 1536; +} + // Intention: Verify the behavior of extract_json_int when the value is a // floating-point number (3.14). The parser reads digits until it // hits the decimal point and returns the integer portion. @@ -516,6 +528,8 @@ int main() { run("extract_json_int_positive", test_extract_json_int_positive); run("extract_json_int_negative", test_extract_json_int_negative); run("extract_json_int_missing", test_extract_json_int_missing); + run("extract_json_int_nested_only", test_extract_json_int_nested_only_uses_fallback); + run("extract_json_int_top_level_wins", test_extract_json_int_top_level_wins_over_nested); run("extract_json_int_float_value", test_extract_json_int_float_value); run("int_or_first_array_scalar", test_extract_json_int_or_first_array_scalar); run("int_or_first_array_array", test_extract_json_int_or_first_array_array); diff --git a/tests/e2e/models/albert/test_albert_build_engine_integration.py b/tests/e2e/models/albert/test_albert_build_engine_integration.py index 308b6ee807..c320ce4a4d 100644 --- a/tests/e2e/models/albert/test_albert_build_engine_integration.py +++ b/tests/e2e/models/albert/test_albert_build_engine_integration.py @@ -11,6 +11,8 @@ import numpy as np import pytest +pytestmark = [pytest.mark.gpu, pytest.mark.trt] + pytest.importorskip("tensorrt", reason="TensorRT is required for family builder tests") diff --git a/tests/e2e/models/bart/test_bart_build_engine_integration.py b/tests/e2e/models/bart/test_bart_build_engine_integration.py index 568fe5c34a..29726a028c 100644 --- a/tests/e2e/models/bart/test_bart_build_engine_integration.py +++ b/tests/e2e/models/bart/test_bart_build_engine_integration.py @@ -108,6 +108,8 @@ def _make(vocab, hidden, layers, heads, ffn, max_pos): t[f"{pfx}.final_layer_norm.bias"] = _rand(hidden) return t + @pytest.mark.gpu + @pytest.mark.trt def test_build_engine(self, tmp_path): from tensorrt_model_connect.families.bart import plugin config = { diff --git a/tests/e2e/models/codegen/test_codegen_build_engine_integration.py b/tests/e2e/models/codegen/test_codegen_build_engine_integration.py index f03c2835a8..cf9e6c5a9d 100644 --- a/tests/e2e/models/codegen/test_codegen_build_engine_integration.py +++ b/tests/e2e/models/codegen/test_codegen_build_engine_integration.py @@ -11,6 +11,8 @@ import numpy as np import pytest +pytestmark = [pytest.mark.gpu, pytest.mark.trt] + pytest.importorskip("tensorrt", reason="TensorRT is required for family builder tests") diff --git a/tests/e2e/models/deberta/test_deberta_build_engine_integration.py b/tests/e2e/models/deberta/test_deberta_build_engine_integration.py index 66eadf651a..bb875d43d9 100644 --- a/tests/e2e/models/deberta/test_deberta_build_engine_integration.py +++ b/tests/e2e/models/deberta/test_deberta_build_engine_integration.py @@ -11,6 +11,8 @@ import numpy as np import pytest +pytestmark = [pytest.mark.gpu, pytest.mark.trt] + pytest.importorskip("tensorrt", reason="TensorRT is required for family builder tests") diff --git a/tests/e2e/models/deepseek_ocr/test_deepseek_ocr_builder_tp.py b/tests/e2e/models/deepseek_ocr/test_deepseek_ocr_builder_tp.py index dde9c0d525..c5d8024123 100644 --- a/tests/e2e/models/deepseek_ocr/test_deepseek_ocr_builder_tp.py +++ b/tests/e2e/models/deepseek_ocr/test_deepseek_ocr_builder_tp.py @@ -482,6 +482,8 @@ def test_deepseek_ocr_bounds_sequence_prefill_profile( assert sequence_prefill_profile_lengths(max_cache_length) == expected +@pytest.mark.gpu +@pytest.mark.trt def test_deepseek_ocr_decomposes_multirow_sequence_attention() -> None: trt = trt_compat.get_trt() builder = trt.Builder(trt.Logger(trt.Logger.ERROR)) diff --git a/tests/e2e/models/dinov3/test_dinov3_convnext_builder.py b/tests/e2e/models/dinov3/test_dinov3_convnext_builder.py index 963039a87c..23a55cc460 100644 --- a/tests/e2e/models/dinov3/test_dinov3_convnext_builder.py +++ b/tests/e2e/models/dinov3/test_dinov3_convnext_builder.py @@ -22,6 +22,8 @@ def _load_builder(monkeypatch: pytest.MonkeyPatch): return importlib.import_module("tensorrt_model_connect.families.dinov3.convnext_builder") +@pytest.mark.gpu +@pytest.mark.trt @pytest.mark.parametrize("precision", ["fp32", "fp16"]) def test_real_tensorrt_build_marks_hf_outputs(precision: str) -> None: trt = pytest.importorskip("tensorrt") @@ -317,6 +319,8 @@ def test_build_source_is_native_and_marks_hf_outputs( assert "onnx" not in source.lower() +@pytest.mark.gpu +@pytest.mark.trt def test_real_convnext_fp32_matches_transformers(tmp_path) -> None: trt = pytest.importorskip("tensorrt") torch = pytest.importorskip("torch") diff --git a/tests/e2e/models/dinov3/test_dinov3_vit_builder.py b/tests/e2e/models/dinov3/test_dinov3_vit_builder.py index 671e9c0cea..534b42913d 100644 --- a/tests/e2e/models/dinov3/test_dinov3_vit_builder.py +++ b/tests/e2e/models/dinov3/test_dinov3_vit_builder.py @@ -286,6 +286,8 @@ def test_timm_dinov3_qkvb_config_normalizes_before_metadata(tmp_path: Path) -> N assert metadata["num_attention_heads"] == 6 +@pytest.mark.gpu +@pytest.mark.trt def test_real_tensorrt_timm_mapped_vit_build(tmp_path: Path) -> None: _write_tiny_timm_vit(tmp_path) config = ModelConfig.from_dir(tmp_path) @@ -319,6 +321,8 @@ def test_vit_config_and_bundle_metadata_preserve_hf_contract(tmp_path: Path) -> assert plugin.default_max_cache_length(config) == 1 +@pytest.mark.gpu +@pytest.mark.trt @pytest.mark.parametrize("precision", ["fp32", "fp16"]) def test_real_tensorrt_vit_build_marks_hf_outputs(tmp_path: Path, precision: str) -> None: _write_tiny_vit(tmp_path) @@ -337,6 +341,8 @@ def test_real_tensorrt_vit_build_marks_hf_outputs(tmp_path: Path, precision: str assert engine.get_tensor_dtype("pooler_output") == trt.float32 +@pytest.mark.gpu +@pytest.mark.trt def test_real_tensorrt_vit_fp32_matches_transformers(tmp_path: Path) -> None: torch = pytest.importorskip("torch") transformers = pytest.importorskip("transformers") diff --git a/tests/e2e/models/distilbert/test_distilbert_build_engine_integration.py b/tests/e2e/models/distilbert/test_distilbert_build_engine_integration.py index 38f9bbec28..6e75de48dd 100644 --- a/tests/e2e/models/distilbert/test_distilbert_build_engine_integration.py +++ b/tests/e2e/models/distilbert/test_distilbert_build_engine_integration.py @@ -11,6 +11,8 @@ import numpy as np import pytest +pytestmark = [pytest.mark.gpu, pytest.mark.trt] + pytest.importorskip("tensorrt", reason="TensorRT is required for family builder tests") diff --git a/tests/e2e/models/dpr/test_dpr_build_engine_integration.py b/tests/e2e/models/dpr/test_dpr_build_engine_integration.py index 21d0bc816d..db0ef6ef4e 100644 --- a/tests/e2e/models/dpr/test_dpr_build_engine_integration.py +++ b/tests/e2e/models/dpr/test_dpr_build_engine_integration.py @@ -11,6 +11,8 @@ import numpy as np import pytest +pytestmark = [pytest.mark.gpu, pytest.mark.trt] + pytest.importorskip("tensorrt", reason="TensorRT is required for family builder tests") diff --git a/tests/e2e/models/eagle_vlm/test_eagle_vlm_builder_tp.py b/tests/e2e/models/eagle_vlm/test_eagle_vlm_builder_tp.py index f76375a6cd..1c00c70ef4 100644 --- a/tests/e2e/models/eagle_vlm/test_eagle_vlm_builder_tp.py +++ b/tests/e2e/models/eagle_vlm/test_eagle_vlm_builder_tp.py @@ -90,6 +90,8 @@ class Config(_Config): assert plugin_module._resolve_rope_scaling(Config())["factor"] == 8.0 +@pytest.mark.gpu +@pytest.mark.trt def test_eagle_vlm_fp16_reranker_keeps_residual_and_norms_in_fp32( monkeypatch, ) -> None: @@ -223,6 +225,8 @@ class RerankerConfig(_Config): assert mlp_fp32_down_projection == expected_fp32_down +@pytest.mark.gpu +@pytest.mark.trt def test_eagle_vlm_reranker_executes_actual_sequence_length() -> None: import tensorrt as trt diff --git a/tests/e2e/models/electra/test_electra_build_engine_integration.py b/tests/e2e/models/electra/test_electra_build_engine_integration.py index ea76e0f186..529b76d517 100644 --- a/tests/e2e/models/electra/test_electra_build_engine_integration.py +++ b/tests/e2e/models/electra/test_electra_build_engine_integration.py @@ -11,6 +11,8 @@ import numpy as np import pytest +pytestmark = [pytest.mark.gpu, pytest.mark.trt] + pytest.importorskip("tensorrt", reason="TensorRT is required for family builder tests") diff --git a/tests/e2e/models/fnet/test_fnet_build_engine_integration.py b/tests/e2e/models/fnet/test_fnet_build_engine_integration.py index d38b242bc2..a934223cbf 100644 --- a/tests/e2e/models/fnet/test_fnet_build_engine_integration.py +++ b/tests/e2e/models/fnet/test_fnet_build_engine_integration.py @@ -11,6 +11,8 @@ import numpy as np import pytest +pytestmark = [pytest.mark.gpu, pytest.mark.trt] + pytest.importorskip("tensorrt", reason="TensorRT is required for family builder tests") diff --git a/tests/e2e/models/gpt2/test_gpt2_build_engine_integration.py b/tests/e2e/models/gpt2/test_gpt2_build_engine_integration.py index 731f4f1236..4fd2d5b199 100644 --- a/tests/e2e/models/gpt2/test_gpt2_build_engine_integration.py +++ b/tests/e2e/models/gpt2/test_gpt2_build_engine_integration.py @@ -79,6 +79,8 @@ def _make(vocab, hidden, layers, heads, max_pos, mlp): t["lm_head.weight"] = _rand(vocab, hidden) return t + @pytest.mark.gpu + @pytest.mark.trt def test_build_engine(self, tmp_path): from tensorrt_model_connect.families.gpt2 import plugin config = { diff --git a/tests/e2e/models/gpt_neo/test_gpt_neo_build_engine_integration.py b/tests/e2e/models/gpt_neo/test_gpt_neo_build_engine_integration.py index 2b09f2bf2b..784af4f268 100644 --- a/tests/e2e/models/gpt_neo/test_gpt_neo_build_engine_integration.py +++ b/tests/e2e/models/gpt_neo/test_gpt_neo_build_engine_integration.py @@ -11,6 +11,8 @@ import numpy as np import pytest +pytestmark = [pytest.mark.gpu, pytest.mark.trt] + pytest.importorskip("tensorrt", reason="TensorRT is required for family builder tests") diff --git a/tests/e2e/models/gpt_neox/test_gpt_neox_build_engine_integration.py b/tests/e2e/models/gpt_neox/test_gpt_neox_build_engine_integration.py index 886defc1bd..d6a9ab0b5a 100644 --- a/tests/e2e/models/gpt_neox/test_gpt_neox_build_engine_integration.py +++ b/tests/e2e/models/gpt_neox/test_gpt_neox_build_engine_integration.py @@ -11,6 +11,8 @@ import numpy as np import pytest +pytestmark = [pytest.mark.gpu, pytest.mark.trt] + pytest.importorskip("tensorrt", reason="TensorRT is required for family builder tests") diff --git a/tests/e2e/models/gpt_oss/test_gpt_oss_rope_and_sliding.py b/tests/e2e/models/gpt_oss/test_gpt_oss_rope_and_sliding.py index d79c077d0d..0fefc747f2 100644 --- a/tests/e2e/models/gpt_oss/test_gpt_oss_rope_and_sliding.py +++ b/tests/e2e/models/gpt_oss/test_gpt_oss_rope_and_sliding.py @@ -247,6 +247,8 @@ def _require_cuda() -> None: pytest.skip("No CUDA device available for engine execution tests") +@pytest.mark.gpu +@pytest.mark.trt def test_sliding_layers_restrict_attention_to_window(): """Engines with sliding layer_types must diverge from all-full engines exactly when the prompt outgrows the sliding window.""" diff --git a/tests/e2e/models/internlm/test_internlm_build_engine_integration.py b/tests/e2e/models/internlm/test_internlm_build_engine_integration.py index a2198ec508..666d54be2b 100644 --- a/tests/e2e/models/internlm/test_internlm_build_engine_integration.py +++ b/tests/e2e/models/internlm/test_internlm_build_engine_integration.py @@ -11,6 +11,8 @@ import numpy as np import pytest +pytestmark = [pytest.mark.gpu, pytest.mark.trt] + pytest.importorskip("tensorrt", reason="TensorRT is required for family builder tests") diff --git a/tests/e2e/models/m2m_100/test_m2m_100_build_engine_integration.py b/tests/e2e/models/m2m_100/test_m2m_100_build_engine_integration.py index 573281b90f..58ac5b9d6a 100644 --- a/tests/e2e/models/m2m_100/test_m2m_100_build_engine_integration.py +++ b/tests/e2e/models/m2m_100/test_m2m_100_build_engine_integration.py @@ -11,6 +11,8 @@ import numpy as np import pytest +pytestmark = [pytest.mark.gpu, pytest.mark.trt] + pytest.importorskip("tensorrt", reason="TensorRT is required for family builder tests") diff --git a/tests/e2e/models/modernbert/test_modernbert_build_engine_integration.py b/tests/e2e/models/modernbert/test_modernbert_build_engine_integration.py index 7fb884c638..3696b88592 100644 --- a/tests/e2e/models/modernbert/test_modernbert_build_engine_integration.py +++ b/tests/e2e/models/modernbert/test_modernbert_build_engine_integration.py @@ -11,6 +11,8 @@ import numpy as np import pytest +pytestmark = [pytest.mark.gpu, pytest.mark.trt] + pytest.importorskip("tensorrt", reason="TensorRT is required for family builder tests") diff --git a/tests/e2e/models/olmo2/test_olmo2_build_engine_integration.py b/tests/e2e/models/olmo2/test_olmo2_build_engine_integration.py index 7652282740..db5e9456b7 100644 --- a/tests/e2e/models/olmo2/test_olmo2_build_engine_integration.py +++ b/tests/e2e/models/olmo2/test_olmo2_build_engine_integration.py @@ -11,6 +11,8 @@ import numpy as np import pytest +pytestmark = [pytest.mark.gpu, pytest.mark.trt] + pytest.importorskip("tensorrt", reason="TensorRT is required for family builder tests") diff --git a/tests/e2e/models/phi4_multimodal/test_phi4_multimodal_family_plugin.py b/tests/e2e/models/phi4_multimodal/test_phi4_multimodal_family_plugin.py index 86903cf9b5..c2c109d50b 100644 --- a/tests/e2e/models/phi4_multimodal/test_phi4_multimodal_family_plugin.py +++ b/tests/e2e/models/phi4_multimodal/test_phi4_multimodal_family_plugin.py @@ -519,6 +519,8 @@ def test_tied_embeddings(self, tmp_path): w_out = weights["w_out"] np.testing.assert_allclose(w_out, embedding.T, atol=1e-6) + @pytest.mark.gpu + @pytest.mark.trt def test_selected_fp32_decoder_layer_builds_in_fp16_engine(self, tmp_path): from tensorrt_model_connect.families.phi4_multimodal import plugin diff --git a/tests/e2e/models/qwen/test_perf_parity.py b/tests/e2e/models/qwen/test_perf_parity.py index 10dfe612fc..f19b8671f8 100644 --- a/tests/e2e/models/qwen/test_perf_parity.py +++ b/tests/e2e/models/qwen/test_perf_parity.py @@ -24,6 +24,8 @@ import pytest +pytestmark = [pytest.mark.gpu, pytest.mark.trt, pytest.mark.e2e] + PROJECT_DIR = Path(__file__).resolve().parents[4] DEFAULT_ENGINE_DIR = Path("/mnt/storage/tensorrt-model-connect/engines") DEFAULT_BINARY = PROJECT_DIR / "build" / "trtmc" diff --git a/tests/e2e/models/t5/test_t5_build_engine_integration.py b/tests/e2e/models/t5/test_t5_build_engine_integration.py index 1fd2a01967..edeedb1109 100644 --- a/tests/e2e/models/t5/test_t5_build_engine_integration.py +++ b/tests/e2e/models/t5/test_t5_build_engine_integration.py @@ -11,6 +11,8 @@ import numpy as np import pytest +pytestmark = [pytest.mark.gpu, pytest.mark.trt] + pytest.importorskip("tensorrt", reason="TensorRT is required for family builder tests") diff --git a/tests/tools/test_community_ci.py b/tests/tools/test_community_ci.py index fe038bf5ab..8da9aa056a 100644 --- a/tests/tools/test_community_ci.py +++ b/tests/tools/test_community_ci.py @@ -82,6 +82,8 @@ def test_contributor_guides_match_the_live_ci_flow(path: Path) -> None: "py -3 -m pip", ): assert marker in source + if path == REPO_ROOT / "website" / "docs" / "extend" / "contributing.md": + assert "all source-only CPU-safe" in source assert "/run-ci" not in source assert "status comment" not in source @@ -108,7 +110,7 @@ def test_impact_publishes_only_the_public_cpu_scope( lambda *_args, **_kwargs: { "mode": "unit", "run_unit_tests": True, - "unit_scope": "cli", + "unit_scope": "all", "changes": [ { "classifications": [ @@ -138,14 +140,14 @@ def test_impact_publishes_only_the_public_cpu_scope( result = runner.impact(None) - assert result["unit_scope"] == "cli" + assert result["unit_scope"] == "all" assert github_output.read_text(encoding="utf-8") == ( "run_unit_tests=true\n" - "unit_scope=cli\n" + "unit_scope=all\n" 'python_test_targets=["tests/e2e/models/qwen/test_qwen_native_kv_routing.py"]\n' ) summary = github_summary.read_text(encoding="utf-8") - assert "Unit scope: `cli`" in summary + assert "Unit scope: `all`" in summary assert "src/runtime/config/cli_support.cpp" in summary @@ -189,15 +191,22 @@ def test_public_workflow_is_an_automatic_read_only_exact_merge_gate() -> None: assert all(job["runs-on"] == "ubuntu-24.04" for job in jobs.values()) for job_name in ("source-quality", "docs", "ownership-impact", "unit"): assert jobs[job_name]["permissions"] == {"contents": "read"} - assert jobs["unit"]["if"] == "${{ !cancelled() }}" - assert jobs["unit"]["needs"] == "ownership-impact" + assert "if" not in jobs["unit"] + assert "needs" not in jobs["unit"] assert jobs["ownership-impact"]["outputs"]["python_test_targets"] == ( "${{ steps.impact.outputs.python_test_targets }}" ) unit_steps = {step["name"]: step for step in jobs["unit"]["steps"]} - assert unit_steps["Run hardened source-only units"]["env"][ - "TRTMC_PREMERGE_PYTHON_TEST_TARGETS" - ] == "${{ needs.ownership-impact.outputs.python_test_targets }}" + assert list(unit_steps) == [ + "Check out the exact PR merge", + "Set up Python", + "Run hardened source-only units", + ] + assert unit_steps["Run hardened source-only units"] == { + "name": "Run hardened source-only units", + "run": "python3 -m tools.community_ci unit --scope all", + } + assert all("if" not in step for step in unit_steps.values()) assert jobs["required"]["needs"] == [ "source-quality", "docs", @@ -257,7 +266,7 @@ def test_public_workflow_is_an_automatic_read_only_exact_merge_gate() -> None: ("failure", "success", "success", "success", 1), ("success", "failure", "success", "success", 1), ("success", "skipped", "success", "success", 1), - ("success", "success", "failure", "failure", 1), + ("success", "success", "failure", "success", 1), ], ) def test_public_required_job_fails_closed( @@ -306,6 +315,9 @@ def test_cpu_image_installs_the_same_pinned_community_requirements() -> None: assert "COPY community-ci.txt" in dockerfile assert "pip install --requirement /tmp/trtmc-community-ci.txt" in dockerfile assert '"libnvinfer11=${TENSORRT_APT_VERSION}"' in dockerfile + assert "libcurand-dev-13-3" in dockerfile + requirements = (REPO_ROOT / "requirements" / "community-ci.txt").read_text() + assert "pyarrow==25.0.1" in requirements assert "pip install --no-deps" in dockerfile assert '"tensorrt_cu13_bindings==${TENSORRT_VERSION}"' in dockerfile assert '"tensorrt==${TENSORRT_VERSION}"' not in dockerfile diff --git a/tests/tools/test_github_actions_ci.py b/tests/tools/test_github_actions_ci.py index 03ffe58bb8..9e09fb5498 100644 --- a/tests/tools/test_github_actions_ci.py +++ b/tests/tools/test_github_actions_ci.py @@ -1205,7 +1205,7 @@ def test_source_quality_lint_uses_resolved_ci_base_ref() -> None: assert "f\"origin/{self.context.env.get('GITHUB_REF_NAME', 'main')}\"" in text -def test_premerge_unit_stage_builds_no_model_plugins_or_native_wheel() -> None: +def test_premerge_unit_stage_runs_all_cpu_tests_without_native_wheel() -> None: script = _ci_source("quality.py") stage = script.split("def premerge", maxsplit=1)[1].split("def _premerge_scope", maxsplit=1)[0] cmake = (REPO_ROOT / "CMakeLists.txt").read_text() @@ -1216,17 +1216,25 @@ def test_premerge_unit_stage_builds_no_model_plugins_or_native_wheel() -> None: assert '"TRTMC_CI_SCRATCH_DIR", "/tmp"' in stage assert "TRTMC_PREMERGE_UNIT_BUILD_DIR" in stage assert '"not gpu and not trt and not e2e and not model_proof_allocator"' in stage - assert "tests/builder/" in stage - assert "tests/tools/" in stage - assert 'glob("test_*.py")' in script + assert '["tests/builder/", "tests/tools/", "tests/e2e_harness/"]' in script + assert "tests/e2e/models" in script + assert "python/tensorrt_model_connect/families" in script + assert "tests/test_e2e_selection.py" in script + assert "tests/e2e/test_diffusion_image_parity_inputs.py" in script + assert "tests/e2e/test_error_handling.py" in stage + assert '"--ignore-glob=*_e2e.py"' in stage + assert "_mixed_e2e_cpu_contract_files" in script + assert 'f"--deselect={path}::test_model_e2e"' in stage assert '"-q"' in stage and '"-x"' in stage assert '"--dist=worksteal"' in stage + assert '"--import-mode=importlib"' in stage assert 'if scope == "community-all"' in stage assert "test_distinct_explicit_hf_cache_paths_reach_both_containers" in stage assert 'not model_proof_allocator"' in stage assert '"-m"' in stage and '"model_proof_allocator"' in stage assert '["trtmc", "test_cli_args", "test_config_cli_support"]' in script - assert '["trtmc", "trtmc_platform_cpp_tests"]' in script + assert '["trtmc", "trtmc_cpu_cpp_tests"]' in script + assert '["-L", "cpu"]' in script assert '"TRTMC_PREMERGE_UNIT_SCOPE", "all"' in stage assert "tests/builder/test_cli.py" in script assert 'if scope == "builder"' in script @@ -1235,18 +1243,29 @@ def test_premerge_unit_stage_builds_no_model_plugins_or_native_wheel() -> None: assert '[build / "trtmc", "version"]' in stage assert '[build / "trtmc", "--help"]' in stage assert "--stop-on-failure" in stage - assert "libtrtmc_model_*.so*" in stage assert "-DTRTMC_ENABLE_TRT=OFF" not in stage assert "-DTRTMC_BUILD_BACKEND_TRT=OFF" not in stage assert "-DTRTMC_ENABLE_TVM_FFI=OFF" not in stage assert "conan " not in stage assert "build_pip_package" not in stage + assert "--ignore=tests/builder/test_flashinfer_benchmark.py" in stage + assert "--ignore=tests/builder/test_tvm_ffi_plugin.py" in stage assert "trtmc_model_plugins" not in stage assert "add_custom_target(trtmc_platform_cpp_tests)" in cmake + assert "add_custom_target(trtmc_cpu_cpp_tests)" in cmake + assert "if(ARG_UNPARSED_ARGUMENTS)" in cmake + assert ( + "add_dependencies(trtmc_cpu_cpp_tests test_optimized_runtime_bundle_contract)" in cmake + ) + assert len(re.findall(r"(?m)^\s*add_test\(", cmake)) == 2 + assert "add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME})" in cmake + assert "NAME test_optimized_runtime_bundle_contract" in cmake + assert '"${_trtmc_test_owner_label};${_trtmc_test_resource_label}"' in cmake assert "trtmc_add_test(test_model_plugin_loader MODEL_OWNED)" in cmake assert "test_c_abi_runtime_regression" not in cmake assert ( - "test_c_abi_runtime_regression|test_c_abi_runtime_regression.cpp|trtmc_model_qwen|_|_" + "test_c_abi_runtime_regression|test_c_abi_runtime_regression.cpp|" + "trtmc_model_qwen|_|REQUIRES_GPU" in qwen_manifest ) assert "MODEL_OWNED\n ${_trtmc_test_options}" in cmake @@ -1302,6 +1321,97 @@ def run(self, command: list[object], **_kwargs: object) -> subprocess.CompletedP assert not [command for command in context.commands if command[0] in {"cmake", "ctest"}] +def test_all_unit_scope_isolates_shared_and_family_python_suites(tmp_path: Path) -> None: + mixed_file = ( + tmp_path / "tests/e2e/models/example/optimized_adapter/test_example_e2e.py" + ) + mixed_file.parent.mkdir(parents=True) + mixed_file.write_text( + "def test_model_e2e(): pass\n" + "def test_manifest_contract(): pass\n", + encoding="utf-8", + ) + + class RecordingContext: + repository = tmp_path + env = { + "GITHUB_WORKSPACE": str(tmp_path), + "TRTMC_PREMERGE_UNIT_SCOPE": "community-all", + "TRTMC_UNIT_BUILD_JOBS": "8", + "TRTMC_UNIT_TEST_JOBS": "8", + } + + def __init__(self) -> None: + self.commands: list[list[object]] = [] + + def positive_integer(self, value: str, _name: str) -> int: + return int(value) + + def run(self, command: list[object], **_kwargs: object) -> subprocess.CompletedProcess: + self.commands.append(command) + return subprocess.CompletedProcess(command, 0, stdout="", stderr="") + + context = RecordingContext() + UnitTestRunner(context).premerge() + + pytest_commands = [ + command for command in context.commands if command[:3] == ["python", "-m", "pytest"] + ] + source_commands = [ + command + for command in pytest_commands + if "model_proof_allocator" not in command + and "tests/e2e/test_error_handling.py" not in command + ] + assert len(source_commands) == 4 + shared, models, family, mixed = source_commands + assert all( + target in shared for target in ("tests/builder/", "tests/tools/", "tests/e2e_harness/") + ) + assert "tests/e2e/models/" not in shared + assert "--ignore-glob=*_e2e.py" not in shared + assert all( + target in models + for target in ( + "tests/e2e/models/", + "tests/test_e2e_selection.py", + "tests/e2e/test_diffusion_image_parity_inputs.py", + ) + ) + assert "python/tensorrt_model_connect/families/" not in models + assert "--ignore-glob=*_e2e.py" in models + assert "python/tensorrt_model_connect/families/" in family + assert "tests/e2e/models/" not in family + assert "--ignore-glob=*_e2e.py" not in family + relative_mixed = "tests/e2e/models/example/optimized_adapter/test_example_e2e.py" + assert relative_mixed in mixed + assert f"--deselect={relative_mixed}::test_model_e2e" in mixed + assert "--ignore-glob=*_e2e.py" not in mixed + for command in source_commands: + assert "--import-mode=importlib" in command + assert "not gpu and not trt and not e2e and not model_proof_allocator" in command + + build_command = next( + command for command in context.commands if command[:2] == ["cmake", "--build"] + ) + assert "trtmc_cpu_cpp_tests" in build_command + ctest_command = next(command for command in context.commands if command[0] == "ctest") + label_index = ctest_command.index("-L") + assert ctest_command[label_index : label_index + 2] == ["-L", "cpu"] + + +def test_mixed_e2e_cpu_contract_inventory_is_not_hidden() -> None: + class InventoryContext: + repository = REPO_ROOT + + selected = set(UnitTestRunner(InventoryContext())._mixed_e2e_cpu_contract_files()) + + assert { + "tests/e2e/models/fast_foundation_stereo/test_fast_foundation_stereo_e2e.py", + "tests/e2e/models/minimax_h3/test_minimax_h3_e2e.py", + } <= selected + + @pytest.mark.parametrize( "selected_test", [ diff --git a/tests/tools/test_model_ci.py b/tests/tools/test_model_ci.py index 49c88e350f..e1fdbcd044 100644 --- a/tests/tools/test_model_ci.py +++ b/tests/tools/test_model_ci.py @@ -423,7 +423,171 @@ def test_impact_selects_only_model_a(tmp_path: Path) -> None: assert result["fallback_models"] == [] assert result["matrix"] == {"include": [{"model": "model_a", "selection_kind": "direct"}]} assert result["run_unit_tests"] is True - assert result["unit_scope"] == "builder" + assert result["unit_scope"] == "all" + + +@pytest.mark.parametrize("move_nested_contract", (False, True)) +def test_runtime_test_manifest_metadata_runs_units_without_model_proof( + tmp_path: Path, + move_nested_contract: bool, +) -> None: + repo, _ = _make_repo(tmp_path) + manifest = repo / "src/runtime/models/model_a/MODEL.toml" + prefix = ( + 'id = "model_a"\n' + 'runtime_library = "libtrtmc_model_model_a.so"\n' + 'runtime_strategies = ["model_a_runtime"]\n' + ) + test_entry = ' "test_model_a|test_model_a.cpp|_|_|REQUIRES_TRT",\n' + if move_nested_contract: + manifest.write_text( + prefix + + '[validation_profiles]\n' + + 'decoder_debug = ["model_a_runtime"]\n' + + "runtime_tests = [\n" + + test_entry + + "]\n", + encoding="utf-8", + ) + else: + manifest.write_text( + prefix + "runtime_tests = [\n" + test_entry + "]\n", + encoding="utf-8", + ) + base = _commit(repo, "add runtime test metadata") + + head_entry = test_entry.replace("REQUIRES_TRT", "REQUIRES_TRT,REQUIRES_GPU") + suffix = ( + '[validation_profiles]\n' + 'decoder_debug = ["model_a_runtime"]\n' + if move_nested_contract + else "" + ) + manifest.write_text( + prefix + "runtime_tests = [\n" + head_entry + "]\n" + suffix, + encoding="utf-8", + ) + head = _commit(repo, "update runtime test metadata") + + result = _impact(repo, base, head) + + assert result["mode"] == "unit" + assert result["affected_models"] == [] + assert result["direct_models"] == [] + assert result["fallback_models"] == [] + assert result["matrix"] == {"include": []} + assert result["run_unit_tests"] is True + assert result["unit_scope"] == "all" + assert { + classification["kind"] + for change in result["changes"] + for classification in change["classifications"] + } == {"unit_tests"} + + +def test_runtime_manifest_semantic_change_keeps_direct_model_proof(tmp_path: Path) -> None: + repo, _ = _make_repo(tmp_path) + manifest = repo / "src/runtime/models/model_a/MODEL.toml" + manifest.write_text( + 'id = "model_a"\n' + 'runtime_library = "libtrtmc_model_model_a.so"\n' + 'runtime_strategies = ["model_a_runtime"]\n' + 'runtime_tests = ["test_model_a|test_model_a.cpp|_|_|REQUIRES_TRT"]\n', + encoding="utf-8", + ) + base = _commit(repo, "add runtime test metadata") + manifest.write_text( + 'id = "model_a"\n' + 'runtime_library = "libtrtmc_model_model_a_v2.so"\n' + 'runtime_strategies = ["model_a_runtime"]\n' + 'runtime_tests = ' + '["test_model_a|test_model_a.cpp|_|_|REQUIRES_TRT,REQUIRES_GPU"]\n', + encoding="utf-8", + ) + head = _commit(repo, "change runtime library and tests") + + result = _impact(repo, base, head) + + assert result["mode"] == "models" + assert result["affected_models"] == ["model_a"] + assert result["direct_models"] == ["model_a"] + assert result["matrix"] == { + "include": [{"model": "model_a", "selection_kind": "direct"}] + } + + +@pytest.mark.parametrize("module_marker", (False, True)) +def test_pytest_resource_marker_only_change_runs_units_without_model_proof( + tmp_path: Path, + module_marker: bool, +) -> None: + repo, _ = _make_repo(tmp_path) + path = "tests/e2e/models/model_a/test_builder_contract.py" + _write(repo, path, "import pytest\n\ndef test_contract():\n assert True\n") + base = _commit(repo, "add model test contract") + if module_marker: + updated = ( + "import pytest\n\n" + "pytestmark = [pytest.mark.gpu, pytest.mark.trt]\n\n" + "def test_contract():\n assert True\n" + ) + else: + updated = ( + "import pytest\n\n" + "@pytest.mark.gpu\n" + "@pytest.mark.trt\n" + "def test_contract():\n assert True\n" + ) + _write(repo, path, updated) + head = _commit(repo, "classify model test resources") + + result = _impact(repo, base, head) + + assert result["mode"] == "unit" + assert result["affected_models"] == [] + assert result["direct_models"] == [] + assert result["matrix"] == {"include": []} + assert result["unit_scope"] == "all" + + +def test_pytest_marker_and_test_semantic_change_keeps_direct_model_proof( + tmp_path: Path, +) -> None: + repo, _ = _make_repo(tmp_path) + path = "tests/e2e/models/model_a/test_builder_contract.py" + _write(repo, path, "import pytest\n\ndef test_contract():\n assert True\n") + base = _commit(repo, "add model test contract") + _write( + repo, + path, + "import pytest\n\n" + "@pytest.mark.gpu\n" + "def test_contract():\n assert 2 + 2 == 4\n", + ) + head = _commit(repo, "change marker and test behavior") + + result = _impact(repo, base, head) + + assert result["mode"] == "models" + assert result["direct_models"] == ["model_a"] + + +def test_family_python_unit_test_change_does_not_select_model_proof(tmp_path: Path) -> None: + repo, base = _make_repo(tmp_path) + _write( + repo, + "python/tensorrt_model_connect/families/model_a/tests/test_family.py", + "def test_family():\n assert True\n", + ) + head = _commit(repo, "add family unit test") + + result = _impact(repo, base, head) + + assert result["mode"] == "unit" + assert result["affected_models"] == [] + assert result["direct_models"] == [] + assert result["matrix"] == {"include": []} + assert result["unit_scope"] == "all" @pytest.mark.parametrize( @@ -437,7 +601,7 @@ def test_impact_selects_only_model_a(tmp_path: Path) -> None: "src/runtime/models/model_a/model_a.cpp", ), ) -def test_model_owned_change_runs_builder_units( +def test_model_owned_change_runs_all_cpu_units( tmp_path: Path, path: str, ) -> None: @@ -451,10 +615,10 @@ def test_model_owned_change_runs_builder_units( assert result["affected_models"] == ["model_a"] assert result["direct_models"] == ["model_a"] assert result["run_unit_tests"] is True - assert result["unit_scope"] == "builder" + assert result["unit_scope"] == "all" -def test_model_owned_deletion_runs_builder_units(tmp_path: Path) -> None: +def test_model_owned_deletion_runs_all_cpu_units(tmp_path: Path) -> None: repo, _ = _make_repo(tmp_path) path = "python/tensorrt_model_connect/families/model_a/graph_ops.py" _write(repo, path, "# graph implementation\n") @@ -467,7 +631,7 @@ def test_model_owned_deletion_runs_builder_units(tmp_path: Path) -> None: assert result["mode"] == "models" assert result["affected_models"] == ["model_a"] assert result["run_unit_tests"] is True - assert result["unit_scope"] == "builder" + assert result["unit_scope"] == "all" def test_mixed_model_and_cli_change_runs_all_units(tmp_path: Path) -> None: @@ -597,8 +761,8 @@ def test_impact_treats_legal_and_docs_as_no_model_change(tmp_path: Path) -> None assert result["mode"] == "none" assert result["has_models"] is False assert result["matrix"] == {"include": []} - assert result["run_unit_tests"] is False - assert result["unit_scope"] == "none" + assert result["run_unit_tests"] is True + assert result["unit_scope"] == "all" @pytest.mark.parametrize( @@ -606,7 +770,7 @@ def test_impact_treats_legal_and_docs_as_no_model_change(tmp_path: Path) -> None ( ( "website/docs/wiki/Agentic-Quantization-Core-Minimal-Plan.md", - "builder", + "all", "unit_builder", ), ("tools/ci/README.md", "all", "unit_tests"), @@ -801,10 +965,10 @@ def test_impact_treats_shared_family_registry_as_platform(tmp_path: Path) -> Non @pytest.mark.parametrize( "path, expected_scope", ( - ("src/runtime/config/cli_support.cpp", "cli"), - ("include/trtmc/config/cli_support.h", "cli"), - ("python/tensorrt_model_connect/build_cli.py", "cli"), - ("python/tensorrt_model_connect/runtime_config/cli_support.py", "cli"), + ("src/runtime/config/cli_support.cpp", "all"), + ("include/trtmc/config/cli_support.h", "all"), + ("python/tensorrt_model_connect/build_cli.py", "all"), + ("python/tensorrt_model_connect/runtime_config/cli_support.py", "all"), ("tests/builder/test_cli.py", "all"), ("tests/cpp/test_cli_args.cpp", "all"), ("tests/tools/test_cli_contract.py", "all"), diff --git a/tests/tools/test_model_plugin_encapsulation_static.py b/tests/tools/test_model_plugin_encapsulation_static.py index 42a23201ec..796a0320ee 100644 --- a/tests/tools/test_model_plugin_encapsulation_static.py +++ b/tests/tools/test_model_plugin_encapsulation_static.py @@ -17,6 +17,11 @@ import re from pathlib import Path +try: + import tomllib +except ModuleNotFoundError: # Python 3.10 uses the declared tomli dependency. + import tomli as tomllib + from tests.e2e_harness.threshold_policy import requires_threshold_sidecar @@ -1142,6 +1147,52 @@ def test_top_level_cmake_does_not_hardcode_model_owned_cpp_tests() -> None: assert not violations, _format_violations(violations) +def _toml_key_paths( + value: object, + key: str, + path: tuple[str, ...] = (), +) -> list[tuple[str, ...]]: + paths: list[tuple[str, ...]] = [] + if isinstance(value, dict): + for item_key, item in value.items(): + item_path = (*path, item_key) + if item_key == key: + paths.append(item_path) + paths.extend(_toml_key_paths(item, key, item_path)) + elif isinstance(value, list): + for index, item in enumerate(value): + paths.extend(_toml_key_paths(item, key, (*path, str(index)))) + return paths + + +def test_runtime_tests_nested_key_detection_handles_quoted_toml() -> None: + data = tomllib.loads('[model]\n"runtime_tests" = ["test|source|links|extra|options"]\n') + + assert _toml_key_paths(data, "runtime_tests") == [("model", "runtime_tests")] + + +def test_runtime_tests_are_top_level_toml_contracts() -> None: + """Keep CMake discovery and selective model-proof discovery identical.""" + violations = [] + for manifest in sorted(RUNTIME_MODELS.glob("*/MODEL.toml")): + data = tomllib.loads(manifest.read_text(encoding="utf-8")) + runtime_test_paths = _toml_key_paths(data, "runtime_tests") + if any(path != ("runtime_tests",) for path in runtime_test_paths): + violations.append( + ( + manifest, + 0, + "runtime_tests must be top-level, not nested under a preceding TOML table", + ) + ) + continue + entries = data.get("runtime_tests", []) + if not isinstance(entries, list) or not all(isinstance(entry, str) for entry in entries): + violations.append((manifest, 0, "runtime_tests must be a top-level string array")) + + assert not violations, _format_violations(violations) + + def test_shared_chat_template_core_is_model_owned() -> None: """Trace: ARCH-MODPLUG-001 Intent: keep chat template detection and rendering in runtime model-owned files. diff --git a/tools/ci/README.md b/tools/ci/README.md index bfcebbcaba..1c78e8e6e0 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -58,14 +58,15 @@ that enters the run-owned container and invokes `pipeline` there. Opening a pull request or pushing a new commit automatically starts Community CPU against GitHub's exact pull-request merge revision. Source quality, -ownership and impact, and selected source-only units run as separate jobs on -GitHub-hosted public CPU runners. - -Impact analysis also passes directly changed, non-E2E Python test files to the -unit job. The unit job adds those exact targets to its normal scope, so a -model-owned CPU test under `tests/e2e/models/` is not hidden merely because the -normal `builder` scope starts at `tests/builder/`. Pytest still excludes tests -marked `gpu`, `trt`, or `e2e`; the public runner does not execute GPU inference. +ownership and impact, and the complete source-only CPU unit suite run as +separate jobs on GitHub-hosted public CPU runners. The unit job does not wait +for impact analysis: every PR runs all CPU-safe Python tests and all CTests +labeled `cpu`, including model-owned mock and contract tests. Pytest still +excludes tests marked `gpu`, `trt`, or `e2e`; the public runner does not execute +GPU inference. Pure model E2E entrypoints also follow the controlled +`test_*_e2e.py` filename contract. A separate discovery pass reopens any mixed +entrypoint file and runs its CPU contracts while deselecting only the exact +`test_model_e2e` node. The jobs check out only the event's immutable merge SHA with read-only repository permission, no persisted checkout credentials, and no secrets. @@ -106,15 +107,14 @@ the tested merge revision's first parent and the exact tested merge. It emits: - directly affected models; - representative fallback models for shared-platform changes; - the dynamic model matrix; -- whether source-only units are required and their scope. +- the non-selective `all` CPU unit scope used by protected premerge. This is why a model-only change validates that model, while a CI-platform change selects a small representative set instead of all models. -Every model-owned change also selects the `builder` unit scope. That scope runs -the Python `tests/builder/` suite without a native build. CLI-only changes -select `cli`; changes that need both scopes, or any broad source/tooling change, -select `all`. +Every non-empty PR diff emits the `all` CPU unit scope. Community CPU runs that +same scope unconditionally and in parallel with impact analysis. Impact +classification remains selective only for the GPU model-proof matrix. ### 3. Reject cheap failures first diff --git a/tools/ci/quality.py b/tools/ci/quality.py index f899723915..a3c1448904 100644 --- a/tools/ci/quality.py +++ b/tools/ci/quality.py @@ -8,6 +8,7 @@ from __future__ import annotations +import ast import json import shutil from pathlib import Path, PurePosixPath @@ -191,7 +192,18 @@ def premerge(self) -> None: ) scope = self.context.env.get("TRTMC_PREMERGE_UNIT_SCOPE", "all") python_tests, native_targets, ctest_selector = self._premerge_scope(scope) - python_tests.extend(self._selected_python_test_targets(python_tests)) + if scope in {"all", "community-all"}: + self._selected_python_test_targets( + [ + *python_tests, + "tests/e2e/models/", + "python/tensorrt_model_connect/families/", + "tests/test_e2e_selection.py", + "tests/e2e/test_diffusion_image_parity_inputs.py", + ] + ) + else: + python_tests.extend(self._selected_python_test_targets(python_tests)) print(f"Premerge unit scope: {scope}") selected_wheel = SelectedWheelRuntime.prepare( @@ -222,6 +234,7 @@ def premerge(self) -> None: "-n", str(test_jobs), "--dist=worksteal", + "--import-mode=importlib", "-p", "no:cacheprovider", "-m", @@ -236,13 +249,81 @@ def premerge(self) -> None: "--deselect=tests/tools/test_model_proof_runner.py::" "test_distinct_explicit_hf_cache_paths_reach_both_containers" ) - if selected_wheel: - pytest.append("--import-mode=importlib") self.context.run( pytest, limit=self.context.env.get("PYTHON_BUILDER_TIMEOUT", "20m"), updates=python_environment, ) + if scope in {"all", "community-all"}: + self.context.run( + [ + python, + "-m", + "pytest", + "tests/e2e/models/", + "tests/test_e2e_selection.py", + "tests/e2e/test_diffusion_image_parity_inputs.py", + "-q", + "-x", + "-n", + str(test_jobs), + "--dist=worksteal", + "--import-mode=importlib", + "-p", + "no:cacheprovider", + "-m", + "not gpu and not trt and not e2e and not model_proof_allocator", + "--ignore-glob=*_e2e.py", + ], + limit=self.context.env.get("PYTHON_BUILDER_TIMEOUT", "20m"), + updates=python_environment, + ) + self.context.run( + [ + python, + "-m", + "pytest", + "python/tensorrt_model_connect/families/", + "-q", + "-x", + "-n", + str(test_jobs), + "--dist=worksteal", + "--import-mode=importlib", + "-p", + "no:cacheprovider", + "-m", + "not gpu and not trt and not e2e and not model_proof_allocator", + ], + limit=self.context.env.get("PYTHON_BUILDER_TIMEOUT", "20m"), + updates=python_environment, + ) + mixed_e2e_contracts = self._mixed_e2e_cpu_contract_files() + if mixed_e2e_contracts: + e2e_deselectors = [ + f"--deselect={path}::test_model_e2e" for path in mixed_e2e_contracts + ] + self.context.run( + [ + python, + "-m", + "pytest", + *mixed_e2e_contracts, + "-q", + "-x", + "-n", + str(test_jobs), + "--dist=worksteal", + "--import-mode=importlib", + "-p", + "no:cacheprovider", + *e2e_deselectors, + "-m", + "not gpu and not trt and not e2e and not model_proof_allocator", + ], + limit=self.context.env.get("PYTHON_BUILDER_TIMEOUT", "20m"), + updates=python_environment, + ) if scope in {"all", "community-all"}: allocator = [ python, @@ -299,9 +380,6 @@ def premerge(self) -> None: if scope == "cli": self.context.run([build / "trtmc", "version"], limit="1m") self.context.run([build / "trtmc", "--help"], limit="1m") - leaked = next(build.rglob("libtrtmc_model_*.so*"), None) - if leaked: - raise CiError(f"source-only unit build produced a model plugin: {leaked}") self.context.run( [ "ctest", @@ -315,6 +393,26 @@ def premerge(self) -> None: ], limit=self.context.env.get("CPP_UNIT_TIMEOUT", "20m"), ) + if scope in {"all", "community-all"}: + self.context.run( + [ + python, + "-m", + "pytest", + "tests/e2e/test_error_handling.py", + "-q", + "-x", + "-p", + "no:cacheprovider", + "--import-mode=importlib", + "-m", + "not gpu and not trt and not e2e", + "--trtmc-binary", + build / "trtmc", + ], + limit=self.context.env.get("PYTHON_BUILDER_TIMEOUT", "20m"), + updates=python_environment, + ) def _premerge_scope(self, scope: str) -> tuple[list[str], list[str], list[str]]: if scope == "builder": @@ -333,16 +431,10 @@ def _premerge_scope(self, scope: str) -> tuple[list[str], list[str], list[str]]: ["-R", "^(test_cli_args|test_config_cli_support)$"], ) if scope in {"all", "community-all"}: - harness_tests = [ - str(path.relative_to(self.context.repository)) - for path in sorted( - (self.context.repository / "tests/e2e_harness").glob("test_*.py") - ) - ] return ( - ["tests/builder/", "tests/tools/", *harness_tests], - ["trtmc", "trtmc_platform_cpp_tests"], - ["-L", "platform"], + ["tests/builder/", "tests/tools/", "tests/e2e_harness/"], + ["trtmc", "trtmc_cpu_cpp_tests"], + ["-L", "cpu"], ) raise CiError( "TRTMC_PREMERGE_UNIT_SCOPE must be builder, cli, all, or community-all" @@ -400,6 +492,28 @@ def _selected_python_test_targets(self, baseline: list[str]) -> list[str]: print("Additional selected Python tests: " + ", ".join(selected)) return selected + def _mixed_e2e_cpu_contract_files(self) -> list[str]: + root = self.context.repository / "tests" / "e2e" / "models" + selected: list[str] = [] + for path in sorted(root.rglob("test_*_e2e.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + test_names: set[str] = set() + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + if node.name.startswith("test_"): + test_names.add(node.name) + continue + if isinstance(node, ast.ClassDef) and node.name.startswith("Test"): + test_names.update( + child.name + for child in node.body + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)) + and child.name.startswith("test_") + ) + if test_names - {"test_model_e2e"}: + selected.append(str(path.relative_to(self.context.repository))) + return selected + @staticmethod def _covered_by_python_baseline( target: str, diff --git a/tools/model_ci.py b/tools/model_ci.py index bfb8e95eef..8c51e5d15a 100644 --- a/tools/model_ci.py +++ b/tools/model_ci.py @@ -15,6 +15,7 @@ from __future__ import annotations import argparse +import ast import hashlib import json import os @@ -27,6 +28,11 @@ from pathlib import Path, PurePosixPath from typing import Iterable, Sequence +try: + import tomllib +except ModuleNotFoundError: # Python 3.10 uses the declared tomli dependency. + import tomli as tomllib + MODEL_ROOTS = ( "python/tensorrt_model_connect/families", @@ -266,6 +272,7 @@ _MODEL_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*\Z") _MANIFEST_ID_RE = re.compile(r'(?m)^\s*id\s*=\s*"([^"]+)"\s*$') _VALIDATION_OPTIONAL_EXTRA_RE = re.compile(r"validation\s*=\s*\[.*\]\s*\Z") +_PYTEST_RESOURCE_MARKERS = frozenset({"e2e", "gpu", "trt"}) class ModelCIError(RuntimeError): @@ -365,6 +372,201 @@ def _manifest_location(path: str) -> tuple[str, str] | None: return None +def _without_runtime_tests(value: object) -> object: + if isinstance(value, dict): + return { + key: _without_runtime_tests(item) + for key, item in value.items() + if key != "runtime_tests" + } + if isinstance(value, list): + return [_without_runtime_tests(item) for item in value] + return value + + +def _runtime_test_locations( + value: object, + path: tuple[str, ...] = (), +) -> dict[tuple[str, ...], tuple[str, ...]] | None: + locations: dict[tuple[str, ...], tuple[str, ...]] = {} + if isinstance(value, dict): + for key, item in value.items(): + item_path = (*path, key) + if key == "runtime_tests": + if not isinstance(item, list) or not all( + isinstance(entry, str) for entry in item + ): + return None + locations[item_path] = tuple(item) + continue + nested = _runtime_test_locations(item, item_path) + if nested is None: + return None + locations.update(nested) + elif isinstance(value, list): + for index, item in enumerate(value): + nested = _runtime_test_locations(item, (*path, str(index))) + if nested is None: + return None + locations.update(nested) + return locations + + +def _is_runtime_test_manifest_only_change( + repo_root: Path, + change: DiffEntry, + base_entries: dict[str, TreeEntry], + head_entries: dict[str, TreeEntry], +) -> bool: + path = change.new_path + location = _manifest_location(path) if path is not None else None + if ( + change.status != "M" + or path is None + or change.old_path != path + or location is None + or location[0] != "src/runtime/models" + ): + return False + base_entry = base_entries.get(path) + head_entry = head_entries.get(path) + if ( + base_entry is None + or head_entry is None + or base_entry.mode != head_entry.mode + or base_entry.object_type != "blob" + or head_entry.object_type != "blob" + ): + return False + try: + base_data = tomllib.loads(_read_blob(repo_root, base_entry.object_id).decode("utf-8")) + head_data = tomllib.loads(_read_blob(repo_root, head_entry.object_id).decode("utf-8")) + except (UnicodeDecodeError, tomllib.TOMLDecodeError): + return False + base_locations = _runtime_test_locations(base_data) + head_locations = _runtime_test_locations(head_data) + return ( + base_locations is not None + and head_locations is not None + and len(base_locations) <= 1 + and len(head_locations) <= 1 + and all(len(path) <= 2 for path in (*base_locations, *head_locations)) + and base_locations != head_locations + and _without_runtime_tests(base_data) == _without_runtime_tests(head_data) + ) + + +def _pytest_resource_marker_name(node: ast.AST) -> str | None: + if ( + isinstance(node, ast.Attribute) + and node.attr in _PYTEST_RESOURCE_MARKERS + and isinstance(node.value, ast.Attribute) + and node.value.attr == "mark" + and isinstance(node.value.value, ast.Name) + and node.value.value.id == "pytest" + ): + return node.attr + return None + + +class _PytestResourceMarkerStripper(ast.NodeTransformer): + @staticmethod + def _decorators(nodes: list[ast.expr]) -> list[ast.expr]: + return [node for node in nodes if _pytest_resource_marker_name(node) is None] + + def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.AST: + node.decorator_list = self._decorators(node.decorator_list) + return self.generic_visit(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> ast.AST: + node.decorator_list = self._decorators(node.decorator_list) + return self.generic_visit(node) + + def visit_ClassDef(self, node: ast.ClassDef) -> ast.AST: + node.decorator_list = self._decorators(node.decorator_list) + return self.generic_visit(node) + + def visit_Assign(self, node: ast.Assign) -> ast.AST | None: + if ( + len(node.targets) != 1 + or not isinstance(node.targets[0], ast.Name) + or node.targets[0].id != "pytestmark" + ): + return self.generic_visit(node) + if _pytest_resource_marker_name(node.value) is not None: + return None + if isinstance(node.value, (ast.List, ast.Tuple)): + node.value.elts = [ + item + for item in node.value.elts + if _pytest_resource_marker_name(item) is None + ] + if not node.value.elts: + return None + return self.generic_visit(node) + + +def _is_pytest_resource_marker_only_change( + repo_root: Path, + change: DiffEntry, + base_entries: dict[str, TreeEntry], + head_entries: dict[str, TreeEntry], +) -> bool: + path = change.new_path + parts = PurePosixPath(path).parts if path is not None else () + if ( + change.status != "M" + or path is None + or change.old_path != path + or len(parts) != 5 + or parts[:3] != ("tests", "e2e", "models") + or not parts[-1].startswith("test_") + or not parts[-1].endswith(".py") + ): + return False + base_entry = base_entries.get(path) + head_entry = head_entries.get(path) + if ( + base_entry is None + or head_entry is None + or base_entry.mode != head_entry.mode + or base_entry.object_type != "blob" + or head_entry.object_type != "blob" + ): + return False + try: + base_tree = ast.parse( + _read_blob(repo_root, base_entry.object_id).decode("utf-8"), + filename=path, + ) + head_tree = ast.parse( + _read_blob(repo_root, head_entry.object_id).decode("utf-8"), + filename=path, + ) + except (SyntaxError, UnicodeDecodeError): + return False + base_raw = ast.dump(base_tree, include_attributes=False) + head_raw = ast.dump(head_tree, include_attributes=False) + _PytestResourceMarkerStripper().visit(base_tree) + _PytestResourceMarkerStripper().visit(head_tree) + return ( + base_raw != head_raw + and ast.dump(base_tree, include_attributes=False) + == ast.dump(head_tree, include_attributes=False) + ) + + +def _is_family_python_unit_test(path: str) -> bool: + parts = PurePosixPath(path).parts + return ( + len(parts) >= 6 + and parts[:3] == ("python", "tensorrt_model_connect", "families") + and parts[4] == "tests" + and parts[-1].startswith("test_") + and parts[-1].endswith(".py") + ) + + def _validate_model_roots() -> None: roots = [PurePosixPath(root) for root in MODEL_ROOTS] for index, left in enumerate(roots): @@ -1003,6 +1205,8 @@ def calculate_impact( allow_legacy_device_tier=True, ) head_catalog = discover_catalog(repo_root, head, allow_legacy_shared_runtime=True) + base_entries = {entry.path: entry for entry in base_catalog.entries} + head_entries = {entry.path: entry for entry in head_catalog.entries} affected: set[str] = set() fallback_selected: set[str] = set() broad_change = False @@ -1014,6 +1218,20 @@ def calculate_impact( ) serialized_changes: list[dict[str, object]] = [] for change in _diff_entries(repo_root, comparison_base, head_sha): + unit_test_metadata_only = ( + _is_runtime_test_manifest_only_change( + repo_root, + change, + base_entries, + head_entries, + ) + or _is_pytest_resource_marker_only_change( + repo_root, + change, + base_entries, + head_entries, + ) + ) classifications: list[dict[str, object]] = [] path_catalogs = ( (change.old_path, base_catalog), @@ -1025,6 +1243,8 @@ def calculate_impact( continue seen_path_revision.add((path, catalog.revision)) kind, owner = _classify_path(path, catalog) + if unit_test_metadata_only or _is_family_python_unit_test(path): + kind, owner = "unit_tests", None if path == "pyproject.toml" and pyproject_validation_only: kind = "unit_tests" item = {"path": path, "kind": kind} @@ -1055,6 +1275,10 @@ def calculate_impact( } ) direct_affected = set(affected) + # CPU unit tests are intentionally not selective. Impact analysis still + # owns the model-proof matrix, but every non-empty PR diff frontloads the + # complete source-only CPU suite before any selective GPU proof starts. + frontload_cpu_units = bool(serialized_changes) if ( affected - set(head_catalog.models) or affected.intersection(base_catalog.legacy_shared_runtime) @@ -1104,8 +1328,8 @@ def calculate_impact( matrix_models=matrix_models, direct_models=direct_affected, fallback_models=fallback_selected, - run_unit_tests=unit_scope != "none" or broad_change, - unit_scope="all" if broad_change else unit_scope, + run_unit_tests=frontload_cpu_units, + unit_scope="all" if frontload_cpu_units else "none", ) result["base_revision"] = base_catalog.revision result["head_revision"] = head_catalog.revision diff --git a/website/docs/extend/contributing.md b/website/docs/extend/contributing.md index 9729ecd6bc..1d67e93ec3 100644 --- a/website/docs/extend/contributing.md +++ b/website/docs/extend/contributing.md @@ -51,7 +51,7 @@ depending on host-installed binaries. The local hook intentionally stays lightweight and does not build the CLI or run the complete CPU suite. After pushing, the pull request automatically runs -source quality, ownership analysis, and the selected source-only C++ and Python +source quality, ownership analysis, and all source-only CPU-safe C++ and Python units on GitHub-hosted public CPU runners. The protected suite retains the filesystem-specific cache-reflink contract that public runners cannot portably execute. @@ -151,12 +151,15 @@ performance, and release qualification are different evidence tiers. Opening the pull request or pushing a new commit automatically starts contributor-visible, GitHub-hosted `Community CPU` validation against GitHub's exact pull-request merge revision. Separate jobs run source quality, ownership -and impact, and source-only C++ and Python units. No comment or maintainer action -is required. - -For directly changed Python unit tests, impact analysis also passes the exact -test files to the CPU unit job. This includes non-E2E model-owned tests under -`tests/e2e/models/`; tests marked `gpu`, `trt`, or `e2e` remain excluded. +and impact, and all source-only CPU-safe C++ and Python units. No comment or +maintainer action is required. + +The CPU unit job is intentionally non-selective: every pull request runs all +CPU-safe Python tests and all CTests labeled `cpu`, including model-owned mock +and contract tests. Tests marked `gpu`, `trt`, or `e2e` remain excluded. Pure +model E2E entrypoints use the controlled `test_*_e2e.py` filename contract; a +separate pass still runs any CPU contracts colocated in those files while +deselecting only `test_model_e2e`. The test jobs have read-only repository permission and no access to private runners, secrets, or GPUs, and every public job uses a GitHub-hosted